2024-11-20 16:52:00,965 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-20 16:52:00,978 main DEBUG Took 0.010459 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 16:52:00,978 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 16:52:00,978 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 16:52:00,979 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 16:52:00,981 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 16:52:00,989 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 16:52:01,003 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 16:52:01,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 16:52:01,005 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 16:52:01,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 16:52:01,007 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 16:52:01,007 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 16:52:01,008 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 16:52:01,009 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 16:52:01,009 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 16:52:01,010 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 16:52:01,011 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 16:52:01,011 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 16:52:01,012 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 16:52:01,012 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 16:52:01,013 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 16:52:01,013 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 16:52:01,014 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 16:52:01,014 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 16:52:01,015 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 16:52:01,015 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 16:52:01,016 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 16:52:01,016 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 16:52:01,017 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 16:52:01,017 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 16:52:01,018 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 16:52:01,018 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 16:52:01,020 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 16:52:01,022 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 16:52:01,024 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 16:52:01,025 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 16:52:01,027 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 16:52:01,027 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 16:52:01,037 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 16:52:01,039 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 16:52:01,041 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 16:52:01,042 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 16:52:01,042 main DEBUG createAppenders(={Console}) 2024-11-20 16:52:01,043 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-20 16:52:01,043 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-20 16:52:01,044 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-20 16:52:01,044 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 16:52:01,045 main DEBUG OutputStream closed 2024-11-20 16:52:01,045 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 16:52:01,046 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 16:52:01,046 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-20 16:52:01,131 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 16:52:01,134 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 16:52:01,136 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 16:52:01,137 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 16:52:01,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 16:52:01,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 16:52:01,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 16:52:01,140 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 16:52:01,140 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 16:52:01,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 16:52:01,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 16:52:01,142 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 16:52:01,142 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 16:52:01,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 16:52:01,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 16:52:01,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 16:52:01,144 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 16:52:01,145 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 16:52:01,147 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 16:52:01,148 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-20 16:52:01,148 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 16:52:01,149 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-20T16:52:01,390 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74 2024-11-20 16:52:01,393 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 16:52:01,394 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T16:52:01,402 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-20T16:52:01,425 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T16:52:01,442 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/cluster_e1fea354-106b-cccb-72d5-7b2ff6b355d1, deleteOnExit=true 2024-11-20T16:52:01,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T16:52:01,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/test.cache.data in system properties and HBase conf 2024-11-20T16:52:01,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T16:52:01,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/hadoop.log.dir in system properties and HBase conf 2024-11-20T16:52:01,446 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T16:52:01,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T16:52:01,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T16:52:01,526 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T16:52:01,607 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T16:52:01,611 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T16:52:01,611 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T16:52:01,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T16:52:01,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T16:52:01,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T16:52:01,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T16:52:01,614 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T16:52:01,614 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T16:52:01,615 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T16:52:01,615 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/nfs.dump.dir in system properties and HBase conf 2024-11-20T16:52:01,615 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/java.io.tmpdir in system properties and HBase conf 2024-11-20T16:52:01,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T16:52:01,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T16:52:01,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T16:52:02,633 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T16:52:02,711 INFO [Time-limited test {}] log.Log(170): Logging initialized @2512ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T16:52:02,777 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T16:52:02,832 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T16:52:02,852 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T16:52:02,853 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T16:52:02,854 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T16:52:02,866 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T16:52:02,869 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/hadoop.log.dir/,AVAILABLE} 2024-11-20T16:52:02,870 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T16:52:03,042 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/java.io.tmpdir/jetty-localhost-35887-hadoop-hdfs-3_4_1-tests_jar-_-any-9674856578837102633/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T16:52:03,048 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:35887} 2024-11-20T16:52:03,048 INFO [Time-limited test {}] server.Server(415): Started @2850ms 2024-11-20T16:52:03,520 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T16:52:03,528 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T16:52:03,531 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T16:52:03,531 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T16:52:03,531 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T16:52:03,532 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/hadoop.log.dir/,AVAILABLE} 2024-11-20T16:52:03,533 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T16:52:03,632 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/java.io.tmpdir/jetty-localhost-43359-hadoop-hdfs-3_4_1-tests_jar-_-any-2867193334635257258/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T16:52:03,633 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:43359} 2024-11-20T16:52:03,633 INFO [Time-limited test {}] server.Server(415): Started @3435ms 2024-11-20T16:52:03,684 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T16:52:04,579 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/cluster_e1fea354-106b-cccb-72d5-7b2ff6b355d1/data/data1/current/BP-2108073788-172.17.0.2-1732121522165/current, will proceed with Du for space computation calculation, 2024-11-20T16:52:04,579 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/cluster_e1fea354-106b-cccb-72d5-7b2ff6b355d1/data/data2/current/BP-2108073788-172.17.0.2-1732121522165/current, will proceed with Du for space computation calculation, 2024-11-20T16:52:04,613 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T16:52:04,671 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9ab786f1b9b1d480 with lease ID 0xf2fe595ea25fec94: Processing first storage report for DS-030cc8b6-5d9f-4830-b83f-d248e453ef59 from datanode DatanodeRegistration(127.0.0.1:35935, datanodeUuid=d89f575f-a757-488e-8350-da815eb7d10e, infoPort=37331, infoSecurePort=0, ipcPort=40923, storageInfo=lv=-57;cid=testClusterID;nsid=103397995;c=1732121522165) 2024-11-20T16:52:04,673 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9ab786f1b9b1d480 with lease ID 0xf2fe595ea25fec94: from storage DS-030cc8b6-5d9f-4830-b83f-d248e453ef59 node DatanodeRegistration(127.0.0.1:35935, datanodeUuid=d89f575f-a757-488e-8350-da815eb7d10e, infoPort=37331, infoSecurePort=0, ipcPort=40923, storageInfo=lv=-57;cid=testClusterID;nsid=103397995;c=1732121522165), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-20T16:52:04,673 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9ab786f1b9b1d480 with lease ID 0xf2fe595ea25fec94: Processing first storage report for DS-e9fbc37f-3406-410d-8cbd-287a3e882dcf from datanode DatanodeRegistration(127.0.0.1:35935, datanodeUuid=d89f575f-a757-488e-8350-da815eb7d10e, infoPort=37331, infoSecurePort=0, ipcPort=40923, storageInfo=lv=-57;cid=testClusterID;nsid=103397995;c=1732121522165) 2024-11-20T16:52:04,674 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9ab786f1b9b1d480 with lease ID 0xf2fe595ea25fec94: from storage DS-e9fbc37f-3406-410d-8cbd-287a3e882dcf node DatanodeRegistration(127.0.0.1:35935, datanodeUuid=d89f575f-a757-488e-8350-da815eb7d10e, infoPort=37331, infoSecurePort=0, ipcPort=40923, storageInfo=lv=-57;cid=testClusterID;nsid=103397995;c=1732121522165), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T16:52:04,676 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74 2024-11-20T16:52:04,766 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/cluster_e1fea354-106b-cccb-72d5-7b2ff6b355d1/zookeeper_0, clientPort=58094, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/cluster_e1fea354-106b-cccb-72d5-7b2ff6b355d1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/cluster_e1fea354-106b-cccb-72d5-7b2ff6b355d1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T16:52:04,777 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58094 2024-11-20T16:52:04,793 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T16:52:04,797 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T16:52:05,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741825_1001 (size=7) 2024-11-20T16:52:05,464 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625 with version=8 2024-11-20T16:52:05,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/hbase-staging 2024-11-20T16:52:05,552 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T16:52:05,779 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b60eac9bda43:0 server-side Connection retries=45 2024-11-20T16:52:05,787 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T16:52:05,788 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T16:52:05,793 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T16:52:05,794 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T16:52:05,794 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T16:52:05,927 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T16:52:05,983 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T16:52:05,991 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T16:52:05,995 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T16:52:06,017 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 89125 (auto-detected) 2024-11-20T16:52:06,018 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T16:52:06,035 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33377 2024-11-20T16:52:06,056 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33377 connecting to ZooKeeper ensemble=127.0.0.1:58094 2024-11-20T16:52:06,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:333770x0, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T16:52:06,175 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33377-0x1015a728ccd0000 connected 2024-11-20T16:52:06,271 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T16:52:06,273 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T16:52:06,283 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T16:52:06,287 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625, hbase.cluster.distributed=false 2024-11-20T16:52:06,313 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T16:52:06,317 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33377 2024-11-20T16:52:06,317 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33377 2024-11-20T16:52:06,318 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33377 2024-11-20T16:52:06,318 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33377 2024-11-20T16:52:06,322 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33377 2024-11-20T16:52:06,415 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b60eac9bda43:0 server-side Connection retries=45 2024-11-20T16:52:06,417 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T16:52:06,417 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T16:52:06,417 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T16:52:06,417 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T16:52:06,417 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T16:52:06,420 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T16:52:06,422 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T16:52:06,423 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33929 2024-11-20T16:52:06,425 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33929 connecting to ZooKeeper ensemble=127.0.0.1:58094 2024-11-20T16:52:06,426 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T16:52:06,429 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T16:52:06,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:339290x0, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T16:52:06,454 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33929-0x1015a728ccd0001 connected 2024-11-20T16:52:06,454 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:339290x0, quorum=127.0.0.1:58094, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T16:52:06,458 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T16:52:06,466 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T16:52:06,469 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T16:52:06,475 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T16:52:06,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33929 2024-11-20T16:52:06,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33929 2024-11-20T16:52:06,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33929 2024-11-20T16:52:06,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33929 2024-11-20T16:52:06,478 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33929 2024-11-20T16:52:06,494 DEBUG [M:0;b60eac9bda43:33377 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b60eac9bda43:33377 2024-11-20T16:52:06,495 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b60eac9bda43,33377,1732121525625 2024-11-20T16:52:06,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T16:52:06,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T16:52:06,534 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b60eac9bda43,33377,1732121525625 2024-11-20T16:52:06,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T16:52:06,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T16:52:06,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T16:52:06,566 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T16:52:06,567 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b60eac9bda43,33377,1732121525625 from backup master directory 2024-11-20T16:52:06,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T16:52:06,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b60eac9bda43,33377,1732121525625 2024-11-20T16:52:06,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T16:52:06,578 WARN [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T16:52:06,578 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b60eac9bda43,33377,1732121525625 2024-11-20T16:52:06,581 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T16:52:06,582 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T16:52:06,632 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/hbase.id] with ID: 84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e 2024-11-20T16:52:06,632 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/.tmp/hbase.id 2024-11-20T16:52:06,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741826_1002 (size=42) 2024-11-20T16:52:07,042 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/.tmp/hbase.id]:[hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/hbase.id] 2024-11-20T16:52:07,086 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T16:52:07,091 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T16:52:07,108 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-20T16:52:07,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T16:52:07,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T16:52:07,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741827_1003 (size=196) 2024-11-20T16:52:07,585 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T16:52:07,587 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T16:52:07,600 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:07,604 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T16:52:07,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741828_1004 (size=1189) 2024-11-20T16:52:08,048 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store 2024-11-20T16:52:08,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741829_1005 (size=34) 2024-11-20T16:52:08,472 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T16:52:08,476 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:52:08,477 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T16:52:08,477 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T16:52:08,478 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T16:52:08,479 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T16:52:08,479 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T16:52:08,480 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T16:52:08,481 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732121528477Disabling compacts and flushes for region at 1732121528477Disabling writes for close at 1732121528479 (+2 ms)Writing region close event to WAL at 1732121528480 (+1 ms)Closed at 1732121528480 2024-11-20T16:52:08,484 WARN [master/b60eac9bda43:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/.initializing 2024-11-20T16:52:08,484 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/WALs/b60eac9bda43,33377,1732121525625 2024-11-20T16:52:08,492 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T16:52:08,508 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b60eac9bda43%2C33377%2C1732121525625, suffix=, logDir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/WALs/b60eac9bda43,33377,1732121525625, archiveDir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/oldWALs, maxLogs=10 2024-11-20T16:52:08,529 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/WALs/b60eac9bda43,33377,1732121525625/b60eac9bda43%2C33377%2C1732121525625.1732121528512, exclude list is [], retry=0 2024-11-20T16:52:08,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35935,DS-030cc8b6-5d9f-4830-b83f-d248e453ef59,DISK] 2024-11-20T16:52:08,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T16:52:08,579 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/WALs/b60eac9bda43,33377,1732121525625/b60eac9bda43%2C33377%2C1732121525625.1732121528512 2024-11-20T16:52:08,580 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37331:37331)] 2024-11-20T16:52:08,581 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T16:52:08,581 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:52:08,584 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T16:52:08,585 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T16:52:08,619 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T16:52:08,641 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T16:52:08,644 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:08,647 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T16:52:08,647 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T16:52:08,651 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T16:52:08,651 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:08,652 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:52:08,652 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T16:52:08,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T16:52:08,656 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:08,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:52:08,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T16:52:08,661 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T16:52:08,661 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:08,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:52:08,663 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T16:52:08,666 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T16:52:08,667 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T16:52:08,672 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T16:52:08,673 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T16:52:08,677 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T16:52:08,681 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T16:52:08,686 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T16:52:08,687 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60284881, jitterRate=-0.10168527066707611}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T16:52:08,693 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732121528596Initializing all the Stores at 1732121528598 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732121528599 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121528600 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121528600Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121528600Cleaning up temporary data from old regions at 1732121528673 (+73 ms)Region opened successfully at 1732121528692 (+19 ms) 2024-11-20T16:52:08,694 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T16:52:08,722 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d03a7d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b60eac9bda43/172.17.0.2:0 2024-11-20T16:52:08,749 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T16:52:08,759 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T16:52:08,759 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T16:52:08,762 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T16:52:08,763 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T16:52:08,767 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T16:52:08,767 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T16:52:08,790 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T16:52:08,798 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T16:52:08,856 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T16:52:08,859 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T16:52:08,860 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T16:52:08,869 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T16:52:08,871 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T16:52:08,874 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T16:52:08,881 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T16:52:08,882 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T16:52:08,889 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T16:52:08,904 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T16:52:08,914 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T16:52:08,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T16:52:08,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T16:52:08,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T16:52:08,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T16:52:08,930 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b60eac9bda43,33377,1732121525625, sessionid=0x1015a728ccd0000, setting cluster-up flag (Was=false) 2024-11-20T16:52:08,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T16:52:08,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T16:52:08,981 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T16:52:08,983 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b60eac9bda43,33377,1732121525625 2024-11-20T16:52:09,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T16:52:09,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T16:52:09,027 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T16:52:09,029 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b60eac9bda43,33377,1732121525625 2024-11-20T16:52:09,035 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T16:52:09,083 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(746): ClusterId : 84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e 2024-11-20T16:52:09,086 DEBUG [RS:0;b60eac9bda43:33929 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T16:52:09,108 DEBUG [RS:0;b60eac9bda43:33929 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T16:52:09,109 DEBUG [RS:0;b60eac9bda43:33929 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T16:52:09,109 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T16:52:09,118 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T16:52:09,120 DEBUG [RS:0;b60eac9bda43:33929 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T16:52:09,121 DEBUG [RS:0;b60eac9bda43:33929 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50a2ada5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b60eac9bda43/172.17.0.2:0 2024-11-20T16:52:09,124 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T16:52:09,129 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b60eac9bda43,33377,1732121525625 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T16:52:09,135 DEBUG [RS:0;b60eac9bda43:33929 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b60eac9bda43:33929 2024-11-20T16:52:09,136 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b60eac9bda43:0, corePoolSize=5, maxPoolSize=5 2024-11-20T16:52:09,136 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b60eac9bda43:0, corePoolSize=5, maxPoolSize=5 2024-11-20T16:52:09,136 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b60eac9bda43:0, corePoolSize=5, maxPoolSize=5 2024-11-20T16:52:09,136 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b60eac9bda43:0, corePoolSize=5, maxPoolSize=5 2024-11-20T16:52:09,137 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b60eac9bda43:0, corePoolSize=10, maxPoolSize=10 2024-11-20T16:52:09,137 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T16:52:09,137 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b60eac9bda43:0, corePoolSize=2, maxPoolSize=2 2024-11-20T16:52:09,137 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T16:52:09,138 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732121559138 2024-11-20T16:52:09,139 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T16:52:09,139 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T16:52:09,139 DEBUG [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T16:52:09,140 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T16:52:09,141 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T16:52:09,142 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(2659): reportForDuty to master=b60eac9bda43,33377,1732121525625 with port=33929, startcode=1732121526384 2024-11-20T16:52:09,142 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T16:52:09,143 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T16:52:09,146 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T16:52:09,146 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T16:52:09,147 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T16:52:09,147 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T16:52:09,149 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:09,148 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:09,150 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T16:52:09,152 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T16:52:09,153 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T16:52:09,154 DEBUG [RS:0;b60eac9bda43:33929 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T16:52:09,154 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T16:52:09,157 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T16:52:09,158 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T16:52:09,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741831_1007 (size=1321) 2024-11-20T16:52:09,162 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b60eac9bda43:0:becomeActiveMaster-HFileCleaner.large.0-1732121529159,5,FailOnTimeoutGroup] 2024-11-20T16:52:09,163 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b60eac9bda43:0:becomeActiveMaster-HFileCleaner.small.0-1732121529163,5,FailOnTimeoutGroup] 2024-11-20T16:52:09,163 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:09,164 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T16:52:09,165 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:09,165 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:09,193 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59855, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T16:52:09,199 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33377 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b60eac9bda43,33929,1732121526384 2024-11-20T16:52:09,201 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33377 {}] master.ServerManager(517): Registering regionserver=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:09,215 DEBUG [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625 2024-11-20T16:52:09,215 DEBUG [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45599 2024-11-20T16:52:09,215 DEBUG [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T16:52:09,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T16:52:09,245 DEBUG [RS:0;b60eac9bda43:33929 {}] zookeeper.ZKUtil(111): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b60eac9bda43,33929,1732121526384 2024-11-20T16:52:09,245 WARN [RS:0;b60eac9bda43:33929 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T16:52:09,245 INFO [RS:0;b60eac9bda43:33929 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T16:52:09,246 DEBUG [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/WALs/b60eac9bda43,33929,1732121526384 2024-11-20T16:52:09,248 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b60eac9bda43,33929,1732121526384] 2024-11-20T16:52:09,275 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T16:52:09,295 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T16:52:09,300 INFO [RS:0;b60eac9bda43:33929 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T16:52:09,300 INFO [RS:0;b60eac9bda43:33929 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:09,301 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T16:52:09,308 INFO [RS:0;b60eac9bda43:33929 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T16:52:09,309 INFO [RS:0;b60eac9bda43:33929 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:09,309 DEBUG [RS:0;b60eac9bda43:33929 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T16:52:09,310 DEBUG [RS:0;b60eac9bda43:33929 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T16:52:09,310 DEBUG [RS:0;b60eac9bda43:33929 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T16:52:09,310 DEBUG [RS:0;b60eac9bda43:33929 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T16:52:09,310 DEBUG [RS:0;b60eac9bda43:33929 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T16:52:09,310 DEBUG [RS:0;b60eac9bda43:33929 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0, corePoolSize=2, maxPoolSize=2 2024-11-20T16:52:09,311 DEBUG [RS:0;b60eac9bda43:33929 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T16:52:09,311 DEBUG [RS:0;b60eac9bda43:33929 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T16:52:09,311 DEBUG [RS:0;b60eac9bda43:33929 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T16:52:09,311 DEBUG [RS:0;b60eac9bda43:33929 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T16:52:09,311 DEBUG [RS:0;b60eac9bda43:33929 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T16:52:09,312 DEBUG [RS:0;b60eac9bda43:33929 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T16:52:09,312 DEBUG [RS:0;b60eac9bda43:33929 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b60eac9bda43:0, corePoolSize=3, maxPoolSize=3 2024-11-20T16:52:09,312 DEBUG [RS:0;b60eac9bda43:33929 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0, corePoolSize=3, maxPoolSize=3 2024-11-20T16:52:09,314 INFO [RS:0;b60eac9bda43:33929 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:09,314 INFO [RS:0;b60eac9bda43:33929 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:09,314 INFO [RS:0;b60eac9bda43:33929 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:09,314 INFO [RS:0;b60eac9bda43:33929 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:09,314 INFO [RS:0;b60eac9bda43:33929 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:09,315 INFO [RS:0;b60eac9bda43:33929 {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,33929,1732121526384-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T16:52:09,334 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T16:52:09,336 INFO [RS:0;b60eac9bda43:33929 {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,33929,1732121526384-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:09,336 INFO [RS:0;b60eac9bda43:33929 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:09,336 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.Replication(171): b60eac9bda43,33929,1732121526384 started 2024-11-20T16:52:09,354 INFO [RS:0;b60eac9bda43:33929 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:09,355 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(1482): Serving as b60eac9bda43,33929,1732121526384, RpcServer on b60eac9bda43/172.17.0.2:33929, sessionid=0x1015a728ccd0001 2024-11-20T16:52:09,355 DEBUG [RS:0;b60eac9bda43:33929 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T16:52:09,356 DEBUG [RS:0;b60eac9bda43:33929 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b60eac9bda43,33929,1732121526384 2024-11-20T16:52:09,356 DEBUG [RS:0;b60eac9bda43:33929 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b60eac9bda43,33929,1732121526384' 2024-11-20T16:52:09,356 DEBUG [RS:0;b60eac9bda43:33929 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T16:52:09,357 DEBUG [RS:0;b60eac9bda43:33929 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T16:52:09,358 DEBUG [RS:0;b60eac9bda43:33929 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T16:52:09,358 DEBUG [RS:0;b60eac9bda43:33929 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T16:52:09,358 DEBUG [RS:0;b60eac9bda43:33929 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b60eac9bda43,33929,1732121526384 2024-11-20T16:52:09,358 DEBUG [RS:0;b60eac9bda43:33929 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b60eac9bda43,33929,1732121526384' 2024-11-20T16:52:09,358 DEBUG [RS:0;b60eac9bda43:33929 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T16:52:09,359 DEBUG [RS:0;b60eac9bda43:33929 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T16:52:09,360 DEBUG [RS:0;b60eac9bda43:33929 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T16:52:09,360 INFO [RS:0;b60eac9bda43:33929 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T16:52:09,360 INFO [RS:0;b60eac9bda43:33929 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T16:52:09,465 INFO [RS:0;b60eac9bda43:33929 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T16:52:09,468 INFO [RS:0;b60eac9bda43:33929 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b60eac9bda43%2C33929%2C1732121526384, suffix=, logDir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/WALs/b60eac9bda43,33929,1732121526384, archiveDir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/oldWALs, maxLogs=32 2024-11-20T16:52:09,483 DEBUG [RS:0;b60eac9bda43:33929 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/WALs/b60eac9bda43,33929,1732121526384/b60eac9bda43%2C33929%2C1732121526384.1732121529471, exclude list is [], retry=0 2024-11-20T16:52:09,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35935,DS-030cc8b6-5d9f-4830-b83f-d248e453ef59,DISK] 2024-11-20T16:52:09,493 INFO [RS:0;b60eac9bda43:33929 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/WALs/b60eac9bda43,33929,1732121526384/b60eac9bda43%2C33929%2C1732121526384.1732121529471 2024-11-20T16:52:09,493 DEBUG [RS:0;b60eac9bda43:33929 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37331:37331)] 2024-11-20T16:52:09,563 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T16:52:09,564 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625 2024-11-20T16:52:09,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741833_1009 (size=32) 2024-11-20T16:52:09,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:52:09,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T16:52:09,980 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T16:52:09,981 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:09,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T16:52:09,982 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T16:52:09,984 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T16:52:09,984 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:09,985 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T16:52:09,985 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T16:52:09,988 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T16:52:09,988 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:09,989 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T16:52:09,990 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T16:52:09,992 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T16:52:09,992 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:09,993 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T16:52:09,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T16:52:09,995 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740 2024-11-20T16:52:09,996 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740 2024-11-20T16:52:09,999 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T16:52:09,999 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T16:52:10,000 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T16:52:10,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T16:52:10,006 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T16:52:10,007 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69332904, jitterRate=0.03314077854156494}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T16:52:10,009 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732121529975Initializing all the Stores at 1732121529977 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732121529977Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732121529978 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121529978Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732121529978Cleaning up temporary data from old regions at 1732121529999 (+21 ms)Region opened successfully at 1732121530009 (+10 ms) 2024-11-20T16:52:10,009 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T16:52:10,009 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T16:52:10,009 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T16:52:10,009 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T16:52:10,010 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T16:52:10,011 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T16:52:10,011 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732121530009Disabling compacts and flushes for region at 1732121530009Disabling writes for close at 1732121530009Writing region close event to WAL at 1732121530010 (+1 ms)Closed at 1732121530011 (+1 ms) 2024-11-20T16:52:10,014 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T16:52:10,014 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T16:52:10,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T16:52:10,027 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T16:52:10,030 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T16:52:10,183 DEBUG [b60eac9bda43:33377 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-20T16:52:10,193 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:10,199 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,33929,1732121526384, state=OPENING 2024-11-20T16:52:10,248 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T16:52:10,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T16:52:10,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T16:52:10,257 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T16:52:10,257 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T16:52:10,259 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T16:52:10,261 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:52:10,433 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T16:52:10,435 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38773, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T16:52:10,448 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T16:52:10,448 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T16:52:10,449 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T16:52:10,453 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b60eac9bda43%2C33929%2C1732121526384.meta, suffix=.meta, logDir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/WALs/b60eac9bda43,33929,1732121526384, archiveDir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/oldWALs, maxLogs=32 2024-11-20T16:52:10,474 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/WALs/b60eac9bda43,33929,1732121526384/b60eac9bda43%2C33929%2C1732121526384.meta.1732121530456.meta, exclude list is [], retry=0 2024-11-20T16:52:10,478 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35935,DS-030cc8b6-5d9f-4830-b83f-d248e453ef59,DISK] 2024-11-20T16:52:10,481 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/WALs/b60eac9bda43,33929,1732121526384/b60eac9bda43%2C33929%2C1732121526384.meta.1732121530456.meta 2024-11-20T16:52:10,481 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37331:37331)] 2024-11-20T16:52:10,482 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T16:52:10,483 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T16:52:10,486 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T16:52:10,490 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T16:52:10,493 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T16:52:10,494 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:52:10,494 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T16:52:10,494 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T16:52:10,497 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T16:52:10,499 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T16:52:10,499 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:10,500 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T16:52:10,500 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T16:52:10,502 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T16:52:10,502 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:10,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T16:52:10,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T16:52:10,505 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T16:52:10,505 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:10,506 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T16:52:10,506 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T16:52:10,508 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T16:52:10,508 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:10,509 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T16:52:10,509 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T16:52:10,510 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740 2024-11-20T16:52:10,513 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740 2024-11-20T16:52:10,515 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T16:52:10,516 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T16:52:10,516 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T16:52:10,519 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T16:52:10,521 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72114372, jitterRate=0.074587881565094}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T16:52:10,521 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T16:52:10,522 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732121530494Writing region info on filesystem at 1732121530495 (+1 ms)Initializing all the Stores at 1732121530497 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732121530497Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732121530497Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121530497Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732121530497Cleaning up temporary data from old regions at 1732121530516 (+19 ms)Running coprocessor post-open hooks at 1732121530521 (+5 ms)Region opened successfully at 1732121530522 (+1 ms) 2024-11-20T16:52:10,529 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732121530425 2024-11-20T16:52:10,540 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T16:52:10,540 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T16:52:10,542 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:10,546 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,33929,1732121526384, state=OPEN 2024-11-20T16:52:10,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T16:52:10,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T16:52:10,589 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T16:52:10,589 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T16:52:10,590 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:10,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T16:52:10,596 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,33929,1732121526384 in 330 msec 2024-11-20T16:52:10,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T16:52:10,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 578 msec 2024-11-20T16:52:10,605 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T16:52:10,606 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T16:52:10,628 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:10,629 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:10,653 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:10,656 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44423, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:10,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6050 sec 2024-11-20T16:52:10,678 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732121530678, completionTime=-1 2024-11-20T16:52:10,681 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-20T16:52:10,681 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T16:52:10,735 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-20T16:52:10,735 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732121590735 2024-11-20T16:52:10,735 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732121650735 2024-11-20T16:52:10,735 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 54 msec 2024-11-20T16:52:10,739 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,33377,1732121525625-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:10,740 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,33377,1732121525625-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:10,740 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,33377,1732121525625-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:10,749 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b60eac9bda43:33377, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:10,755 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:10,758 DEBUG [master/b60eac9bda43:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T16:52:10,759 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:10,792 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.213sec 2024-11-20T16:52:10,793 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T16:52:10,795 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T16:52:10,796 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T16:52:10,797 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T16:52:10,797 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T16:52:10,798 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,33377,1732121525625-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T16:52:10,799 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,33377,1732121525625-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T16:52:10,808 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T16:52:10,809 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T16:52:10,809 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,33377,1732121525625-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T16:52:10,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:10,895 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T16:52:10,895 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T16:52:10,898 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:10,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:10,908 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:10,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c924b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:10,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:10,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:10,918 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:10,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:10,922 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:10,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:10,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b60eac9bda43,33377,1732121525625 2024-11-20T16:52:10,965 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=311, ProcessCount=11, AvailableMemoryMB=7365 2024-11-20T16:52:10,975 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T16:52:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b60eac9bda43,33377,1732121525625 2024-11-20T16:52:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54ffeb4e 2024-11-20T16:52:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T16:52:10,988 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T16:52:10,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T16:52:10,995 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T16:52:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T16:52:11,007 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T16:52:11,009 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-20T16:52:11,010 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:11,012 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T16:52:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T16:52:11,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741835_1011 (size=1061) 2024-11-20T16:52:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T16:52:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T16:52:11,428 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625 2024-11-20T16:52:11,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741836_1012 (size=53) 2024-11-20T16:52:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T16:52:11,838 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:52:11,838 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b683f7af58301e87dbf6f3d58dd983d2, disabling compactions & flushes 2024-11-20T16:52:11,838 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:11,838 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:11,839 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. after waiting 0 ms 2024-11-20T16:52:11,839 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:11,839 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:11,839 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b683f7af58301e87dbf6f3d58dd983d2: Waiting for close lock at 1732121531838Disabling compacts and flushes for region at 1732121531838Disabling writes for close at 1732121531839 (+1 ms)Writing region close event to WAL at 1732121531839Closed at 1732121531839 2024-11-20T16:52:11,841 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T16:52:11,847 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732121531841"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732121531841"}]},"ts":"1732121531841"} 2024-11-20T16:52:11,852 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T16:52:11,854 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T16:52:11,856 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121531854"}]},"ts":"1732121531854"} 2024-11-20T16:52:11,861 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T16:52:11,862 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b683f7af58301e87dbf6f3d58dd983d2, ASSIGN}] 2024-11-20T16:52:11,865 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b683f7af58301e87dbf6f3d58dd983d2, ASSIGN 2024-11-20T16:52:11,867 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b683f7af58301e87dbf6f3d58dd983d2, ASSIGN; state=OFFLINE, location=b60eac9bda43,33929,1732121526384; forceNewPlan=false, retain=false 2024-11-20T16:52:12,018 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b683f7af58301e87dbf6f3d58dd983d2, regionState=OPENING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:12,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b683f7af58301e87dbf6f3d58dd983d2, ASSIGN because future has completed 2024-11-20T16:52:12,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:52:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T16:52:12,182 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:12,183 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} 2024-11-20T16:52:12,183 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:12,183 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:52:12,183 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:12,183 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:12,186 INFO [StoreOpener-b683f7af58301e87dbf6f3d58dd983d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:12,189 INFO [StoreOpener-b683f7af58301e87dbf6f3d58dd983d2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:52:12,190 INFO [StoreOpener-b683f7af58301e87dbf6f3d58dd983d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b683f7af58301e87dbf6f3d58dd983d2 columnFamilyName A 2024-11-20T16:52:12,190 DEBUG [StoreOpener-b683f7af58301e87dbf6f3d58dd983d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:12,191 INFO [StoreOpener-b683f7af58301e87dbf6f3d58dd983d2-1 {}] regionserver.HStore(327): Store=b683f7af58301e87dbf6f3d58dd983d2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:52:12,192 INFO [StoreOpener-b683f7af58301e87dbf6f3d58dd983d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:12,194 INFO [StoreOpener-b683f7af58301e87dbf6f3d58dd983d2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:52:12,194 INFO [StoreOpener-b683f7af58301e87dbf6f3d58dd983d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b683f7af58301e87dbf6f3d58dd983d2 columnFamilyName B 2024-11-20T16:52:12,194 DEBUG [StoreOpener-b683f7af58301e87dbf6f3d58dd983d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:12,195 INFO [StoreOpener-b683f7af58301e87dbf6f3d58dd983d2-1 {}] regionserver.HStore(327): Store=b683f7af58301e87dbf6f3d58dd983d2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:52:12,196 INFO [StoreOpener-b683f7af58301e87dbf6f3d58dd983d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:12,198 INFO [StoreOpener-b683f7af58301e87dbf6f3d58dd983d2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:52:12,198 INFO [StoreOpener-b683f7af58301e87dbf6f3d58dd983d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b683f7af58301e87dbf6f3d58dd983d2 columnFamilyName C 2024-11-20T16:52:12,198 DEBUG [StoreOpener-b683f7af58301e87dbf6f3d58dd983d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:12,199 INFO [StoreOpener-b683f7af58301e87dbf6f3d58dd983d2-1 {}] regionserver.HStore(327): Store=b683f7af58301e87dbf6f3d58dd983d2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:52:12,200 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:12,200 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:12,201 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:12,202 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:12,204 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:12,204 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:12,205 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T16:52:12,207 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:12,210 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T16:52:12,211 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b683f7af58301e87dbf6f3d58dd983d2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61714326, jitterRate=-0.08038488030433655}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T16:52:12,211 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:12,212 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b683f7af58301e87dbf6f3d58dd983d2: Running coprocessor pre-open hook at 1732121532183Writing region info on filesystem at 1732121532183Initializing all the Stores at 1732121532185 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121532185Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121532185Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121532186 (+1 ms)Cleaning up temporary data from old regions at 1732121532204 (+18 ms)Running coprocessor post-open hooks at 1732121532211 (+7 ms)Region opened successfully at 1732121532212 (+1 ms) 2024-11-20T16:52:12,215 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., pid=6, masterSystemTime=1732121532176 2024-11-20T16:52:12,218 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:12,219 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:12,220 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b683f7af58301e87dbf6f3d58dd983d2, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:12,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 because future has completed 2024-11-20T16:52:12,230 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T16:52:12,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 in 204 msec 2024-11-20T16:52:12,235 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T16:52:12,235 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b683f7af58301e87dbf6f3d58dd983d2, ASSIGN in 368 msec 2024-11-20T16:52:12,237 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T16:52:12,237 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121532237"}]},"ts":"1732121532237"} 2024-11-20T16:52:12,240 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T16:52:12,242 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T16:52:12,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2440 sec 2024-11-20T16:52:13,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T16:52:13,178 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:13,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,182 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:13,183 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:13,184 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46cd3e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:13,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:13,187 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:13,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c82fed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,189 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:13,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:13,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:13,193 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:13,195 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T16:52:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b60eac9bda43,33377,1732121525625 2024-11-20T16:52:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@da7d497 2024-11-20T16:52:13,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T16:52:13,202 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T16:52:13,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,207 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:13,207 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:13,208 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:13,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:13,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:13,212 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:13,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:13,217 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cf4e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:13,221 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,223 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:13,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:13,228 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:13,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:13,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:13,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:13,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:13,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:13,232 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:13,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27108bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,234 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:13,235 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:13,236 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:13,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:13,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:13,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0e6697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:13,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:13,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:13,239 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7ca9c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,241 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:13,242 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:13,243 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70343832, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:13,245 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:13,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f5127c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,251 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:13,251 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:13,252 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:13,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a9c3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:13,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:13,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:13,256 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b096a35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,259 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:13,259 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:13,260 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d111af0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:13,262 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:13,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e9639cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:13,265 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:13,266 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e406351, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:13,269 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44bee5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,274 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:13,275 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:13,276 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17ddb883, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:13,278 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:13,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2171b18c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:13,285 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:13,285 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:13,285 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:13,285 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:13,285 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:13,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:13,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:13,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:13,290 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:13,290 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:13,291 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:13,291 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:13,292 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:13,293 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:13,293 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:13,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:13,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:13,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:13,296 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:13,298 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59498, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:13,298 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:52:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:52:13,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:13,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:52:13,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:52:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:52:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:52:13,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:52:13,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:52:13,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:52:13,325 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-20T16:52:13,345 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T16:52:13,348 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:13,350 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:13,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T16:52:13,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:13,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:13,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:13,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:13,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:13,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T16:52:13,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T16:52:13,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/b827f961f4a94ce2a2e0c411f838bb1c is 50, key is test_row_0/A:col10/1732121533333/Put/seqid=0 2024-11-20T16:52:13,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:13,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:13,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:13,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:13,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:13,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741837_1013 (size=12001) 2024-11-20T16:52:13,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/b827f961f4a94ce2a2e0c411f838bb1c 2024-11-20T16:52:13,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732121533481, max timestamp=1732121533580Num uniques 859; 2024-11-20T16:52:13,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121533481, max timestamp=1732121533580Num uniques 859; 2024-11-20T16:52:13,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:52:13,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:52:13,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732121533481, max timestamp=1732121533580Num uniques 859; 2024-11-20T16:52:13,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:52:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T16:52:13,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T16:52:13,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/ee6f3a44310f4f78ade5979da333ff97 is 50, key is test_row_0/B:col10/1732121533333/Put/seqid=0 2024-11-20T16:52:13,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:13,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:13,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:13,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:13,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:13,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:13,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:13,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121593706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:13,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:13,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121593707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:13,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:13,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121593710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:13,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121593711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:13,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:13,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121593717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:13,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741838_1014 (size=12001) 2024-11-20T16:52:13,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121593867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:13,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:13,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:13,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121593868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121593869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:13,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:13,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T16:52:13,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121593869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:13,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121593876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:13,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:13,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:13,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:13,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:13,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:13,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:13,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:13,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T16:52:14,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T16:52:14,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:14,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:14,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:14,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:14,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:14,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121594087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121594087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121594088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121594099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121594100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/ee6f3a44310f4f78ade5979da333ff97 2024-11-20T16:52:14,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T16:52:14,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:14,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:14,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:14,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:14,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:14,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/87a636d3f5d5485eb44a35267f73a9ea is 50, key is test_row_0/C:col10/1732121533333/Put/seqid=0 2024-11-20T16:52:14,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741839_1015 (size=12001) 2024-11-20T16:52:14,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/87a636d3f5d5485eb44a35267f73a9ea 2024-11-20T16:52:14,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/b827f961f4a94ce2a2e0c411f838bb1c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/b827f961f4a94ce2a2e0c411f838bb1c 2024-11-20T16:52:14,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/b827f961f4a94ce2a2e0c411f838bb1c, entries=150, sequenceid=12, filesize=11.7 K 2024-11-20T16:52:14,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/ee6f3a44310f4f78ade5979da333ff97 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/ee6f3a44310f4f78ade5979da333ff97 2024-11-20T16:52:14,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/ee6f3a44310f4f78ade5979da333ff97, entries=150, sequenceid=12, filesize=11.7 K 2024-11-20T16:52:14,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/87a636d3f5d5485eb44a35267f73a9ea as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/87a636d3f5d5485eb44a35267f73a9ea 2024-11-20T16:52:14,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T16:52:14,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:14,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:14,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/87a636d3f5d5485eb44a35267f73a9ea, entries=150, sequenceid=12, filesize=11.7 K 2024-11-20T16:52:14,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:14,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:14,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:14,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for b683f7af58301e87dbf6f3d58dd983d2 in 988ms, sequenceid=12, compaction requested=false 2024-11-20T16:52:14,386 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T16:52:14,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:14,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-20T16:52:14,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:14,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:14,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:14,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:14,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:14,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:14,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121594432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121594435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121594438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121594444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/5e4703cddd5843d48341f34f664da336 is 50, key is test_row_0/A:col10/1732121533704/Put/seqid=0 2024-11-20T16:52:14,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121594449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T16:52:14,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741840_1016 (size=14341) 2024-11-20T16:52:14,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/5e4703cddd5843d48341f34f664da336 2024-11-20T16:52:14,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T16:52:14,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:14,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:14,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:14,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:14,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:14,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/4107405e3fce409181668b49994bf3ee is 50, key is test_row_0/B:col10/1732121533704/Put/seqid=0 2024-11-20T16:52:14,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121594559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121594559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121594560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121594560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121594566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741841_1017 (size=12001) 2024-11-20T16:52:14,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/4107405e3fce409181668b49994bf3ee 2024-11-20T16:52:14,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/c3242cdf72644333ba6ca3694d6d5f03 is 50, key is test_row_0/C:col10/1732121533704/Put/seqid=0 2024-11-20T16:52:14,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741842_1018 (size=12001) 2024-11-20T16:52:14,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/c3242cdf72644333ba6ca3694d6d5f03 2024-11-20T16:52:14,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/5e4703cddd5843d48341f34f664da336 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/5e4703cddd5843d48341f34f664da336 2024-11-20T16:52:14,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/5e4703cddd5843d48341f34f664da336, entries=200, sequenceid=49, filesize=14.0 K 2024-11-20T16:52:14,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/4107405e3fce409181668b49994bf3ee as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/4107405e3fce409181668b49994bf3ee 2024-11-20T16:52:14,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T16:52:14,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:14,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:14,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:14,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:14,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:14,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/4107405e3fce409181668b49994bf3ee, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T16:52:14,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/c3242cdf72644333ba6ca3694d6d5f03 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/c3242cdf72644333ba6ca3694d6d5f03 2024-11-20T16:52:14,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/c3242cdf72644333ba6ca3694d6d5f03, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T16:52:14,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for b683f7af58301e87dbf6f3d58dd983d2 in 307ms, sequenceid=49, compaction requested=false 2024-11-20T16:52:14,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:14,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T16:52:14,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:14,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:14,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:14,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:14,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:14,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:14,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/637c05243d014b68867fd386acf7131c is 50, key is test_row_0/A:col10/1732121534782/Put/seqid=0 2024-11-20T16:52:14,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T16:52:14,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:14,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:14,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:14,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:14,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741843_1019 (size=12001) 2024-11-20T16:52:14,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:14,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/637c05243d014b68867fd386acf7131c 2024-11-20T16:52:14,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=559, cellSize=25595, totalHeapSize=58376, min timestamp=1732121534811, max timestamp=1732121534859Num uniques 859; 2024-11-20T16:52:14,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=25595, suffix heap size=117064,new segment heap size=58376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=559 2024-11-20T16:52:14,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121534831, max timestamp=1732121534859Num uniques 859; 2024-11-20T16:52:14,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:52:14,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/fccd21c9c51941c2825790380c1c75d4 is 50, key is test_row_0/B:col10/1732121534782/Put/seqid=0 2024-11-20T16:52:14,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121534831, max timestamp=1732121534859Num uniques 859; 2024-11-20T16:52:14,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:52:14,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741844_1020 (size=12001) 2024-11-20T16:52:14,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/fccd21c9c51941c2825790380c1c75d4 2024-11-20T16:52:14,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/bc2a53abcd1844c982828070d6b20647 is 50, key is test_row_0/C:col10/1732121534782/Put/seqid=0 2024-11-20T16:52:14,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121594942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121594943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121594951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121594958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:14,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741845_1021 (size=12001) 2024-11-20T16:52:14,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121594960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T16:52:15,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:15,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:15,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:15,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:15,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:15,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121595068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121595069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121595087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121595087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121595097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T16:52:15,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:15,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:15,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:15,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:15,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:15,269 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T16:52:15,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121595279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121595287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121595296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121595297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T16:52:15,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:15,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:15,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:15,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:15,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:15,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121595307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/bc2a53abcd1844c982828070d6b20647 2024-11-20T16:52:15,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/637c05243d014b68867fd386acf7131c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/637c05243d014b68867fd386acf7131c 2024-11-20T16:52:15,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/637c05243d014b68867fd386acf7131c, entries=150, sequenceid=60, filesize=11.7 K 2024-11-20T16:52:15,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/fccd21c9c51941c2825790380c1c75d4 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/fccd21c9c51941c2825790380c1c75d4 2024-11-20T16:52:15,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/fccd21c9c51941c2825790380c1c75d4, entries=150, sequenceid=60, filesize=11.7 K 2024-11-20T16:52:15,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/bc2a53abcd1844c982828070d6b20647 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/bc2a53abcd1844c982828070d6b20647 2024-11-20T16:52:15,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/bc2a53abcd1844c982828070d6b20647, entries=150, sequenceid=60, filesize=11.7 K 2024-11-20T16:52:15,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T16:52:15,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:15,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:15,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=136.42 KB/139690 for b683f7af58301e87dbf6f3d58dd983d2 in 686ms, sequenceid=60, compaction requested=true 2024-11-20T16:52:15,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:15,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:15,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:15,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:15,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:15,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:15,482 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:15,482 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:15,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:15,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:15,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:15,487 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:15,489 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/B is initiating minor compaction (all files) 2024-11-20T16:52:15,489 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/B in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:15,490 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/ee6f3a44310f4f78ade5979da333ff97, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/4107405e3fce409181668b49994bf3ee, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/fccd21c9c51941c2825790380c1c75d4] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=35.2 K 2024-11-20T16:52:15,491 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting ee6f3a44310f4f78ade5979da333ff97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732121533333 2024-11-20T16:52:15,493 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 4107405e3fce409181668b49994bf3ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732121533677 2024-11-20T16:52:15,493 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:15,493 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/A is initiating minor compaction (all files) 2024-11-20T16:52:15,493 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/A in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:15,494 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/b827f961f4a94ce2a2e0c411f838bb1c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/5e4703cddd5843d48341f34f664da336, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/637c05243d014b68867fd386acf7131c] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=37.4 K 2024-11-20T16:52:15,494 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting fccd21c9c51941c2825790380c1c75d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732121534445 2024-11-20T16:52:15,497 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting b827f961f4a94ce2a2e0c411f838bb1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732121533333 2024-11-20T16:52:15,498 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e4703cddd5843d48341f34f664da336, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732121533677 2024-11-20T16:52:15,499 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 637c05243d014b68867fd386acf7131c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732121534445 2024-11-20T16:52:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T16:52:15,557 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#B#compaction#9 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:15,560 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/085a96f94036497ab14a56a1f592c524 is 50, key is test_row_0/B:col10/1732121534782/Put/seqid=0 2024-11-20T16:52:15,576 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#A#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:15,577 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/f6d7d400f37940ffab44777e3fcbe617 is 50, key is test_row_0/A:col10/1732121534782/Put/seqid=0 2024-11-20T16:52:15,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741846_1022 (size=12104) 2024-11-20T16:52:15,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=143.13 KB heapSize=395.92 KB 2024-11-20T16:52:15,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:15,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:15,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:15,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:15,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:15,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:15,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T16:52:15,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:15,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:15,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:15,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:15,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:15,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741847_1023 (size=12104) 2024-11-20T16:52:15,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/7c1bbb8a57f045d98a0fb9978587233a is 50, key is test_row_0/A:col10/1732121534943/Put/seqid=0 2024-11-20T16:52:15,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121595636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121595647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741848_1024 (size=14341) 2024-11-20T16:52:15,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/7c1bbb8a57f045d98a0fb9978587233a 2024-11-20T16:52:15,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121595651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121595656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121595659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/f657202cb7894d158506a72134f0ba71 is 50, key is test_row_0/B:col10/1732121534943/Put/seqid=0 2024-11-20T16:52:15,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741849_1025 (size=12001) 2024-11-20T16:52:15,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/f657202cb7894d158506a72134f0ba71 2024-11-20T16:52:15,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/79695afbc5bf42b2b9a405beaa4de4a7 is 50, key is test_row_0/C:col10/1732121534943/Put/seqid=0 2024-11-20T16:52:15,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121595767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121595767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741850_1026 (size=12001) 2024-11-20T16:52:15,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/79695afbc5bf42b2b9a405beaa4de4a7 2024-11-20T16:52:15,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T16:52:15,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:15,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:15,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:15,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:15,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:15,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:15,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/7c1bbb8a57f045d98a0fb9978587233a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/7c1bbb8a57f045d98a0fb9978587233a 2024-11-20T16:52:15,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121595786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121595787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121595787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:15,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:15,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/7c1bbb8a57f045d98a0fb9978587233a, entries=200, sequenceid=95, filesize=14.0 K 2024-11-20T16:52:15,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/f657202cb7894d158506a72134f0ba71 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/f657202cb7894d158506a72134f0ba71 2024-11-20T16:52:15,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/f657202cb7894d158506a72134f0ba71, entries=150, sequenceid=95, filesize=11.7 K 2024-11-20T16:52:15,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/79695afbc5bf42b2b9a405beaa4de4a7 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/79695afbc5bf42b2b9a405beaa4de4a7 2024-11-20T16:52:15,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/79695afbc5bf42b2b9a405beaa4de4a7, entries=150, sequenceid=95, filesize=11.7 K 2024-11-20T16:52:15,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~395.88 KB/405376, currentSize=40.25 KB/41220 for b683f7af58301e87dbf6f3d58dd983d2 in 228ms, sequenceid=95, compaction requested=true 2024-11-20T16:52:15,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:A, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T16:52:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:B, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-20T16:52:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-20T16:52:15,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T16:52:15,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:15,943 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-20T16:52:15,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:15,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:15,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:15,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:15,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:15,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:15,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/c66fd4c50ea84713b69ed6e34a3c23e8 is 50, key is test_row_1/A:col10/1732121535655/Put/seqid=0 2024-11-20T16:52:15,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741851_1027 (size=9657) 2024-11-20T16:52:15,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:15,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:16,011 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T16:52:16,014 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/085a96f94036497ab14a56a1f592c524 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/085a96f94036497ab14a56a1f592c524 2024-11-20T16:52:16,041 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/B of b683f7af58301e87dbf6f3d58dd983d2 into 085a96f94036497ab14a56a1f592c524(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:16,041 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:16,047 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/B, priority=13, startTime=1732121535482; duration=0sec 2024-11-20T16:52:16,048 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-20T16:52:16,048 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:B 2024-11-20T16:52:16,048 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-20T16:52:16,055 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T16:52:16,056 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T16:52:16,056 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. because compaction request was cancelled 2024-11-20T16:52:16,056 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:A 2024-11-20T16:52:16,056 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:52:16,060 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/f6d7d400f37940ffab44777e3fcbe617 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/f6d7d400f37940ffab44777e3fcbe617 2024-11-20T16:52:16,060 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:52:16,060 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/C is initiating minor compaction (all files) 2024-11-20T16:52:16,060 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/C in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:16,060 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/87a636d3f5d5485eb44a35267f73a9ea, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/c3242cdf72644333ba6ca3694d6d5f03, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/bc2a53abcd1844c982828070d6b20647, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/79695afbc5bf42b2b9a405beaa4de4a7] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=46.9 K 2024-11-20T16:52:16,062 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 87a636d3f5d5485eb44a35267f73a9ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732121533333 2024-11-20T16:52:16,064 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting c3242cdf72644333ba6ca3694d6d5f03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732121533677 2024-11-20T16:52:16,065 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting bc2a53abcd1844c982828070d6b20647, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732121534445 2024-11-20T16:52:16,066 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 79695afbc5bf42b2b9a405beaa4de4a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732121534943 2024-11-20T16:52:16,083 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/A of b683f7af58301e87dbf6f3d58dd983d2 into f6d7d400f37940ffab44777e3fcbe617(size=11.8 K), total size for store is 25.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:16,083 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:16,084 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/A, priority=13, startTime=1732121535472; duration=0sec 2024-11-20T16:52:16,085 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T16:52:16,085 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:A 2024-11-20T16:52:16,085 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:B 2024-11-20T16:52:16,085 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-20T16:52:16,085 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T16:52:16,085 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T16:52:16,085 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. because compaction request was cancelled 2024-11-20T16:52:16,085 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:C 2024-11-20T16:52:16,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121536038, max timestamp=1732121536064Num uniques 859; 2024-11-20T16:52:16,086 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T16:52:16,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:52:16,087 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T16:52:16,088 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T16:52:16,088 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. because compaction request was cancelled 2024-11-20T16:52:16,088 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:B 2024-11-20T16:52:16,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121536038, max timestamp=1732121536068Num uniques 859; 2024-11-20T16:52:16,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:52:16,117 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#C#compaction#15 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:16,118 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T16:52:16,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121536038, max timestamp=1732121536064Num uniques 859; 2024-11-20T16:52:16,118 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/1349769b0f1b4261b1121a45b1e4e867 is 50, key is test_row_0/C:col10/1732121534943/Put/seqid=0 2024-11-20T16:52:16,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:52:16,118 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T16:52:16,120 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T16:52:16,120 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T16:52:16,121 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T16:52:16,121 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T16:52:16,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T16:52:16,123 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-20T16:52:16,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732121536143, max timestamp=1732121536159Num uniques 1018; 2024-11-20T16:52:16,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732121536143, max timestamp=1732121536159Num uniques 1018; 2024-11-20T16:52:16,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T16:52:16,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T16:52:16,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741852_1028 (size=12139) 2024-11-20T16:52:16,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732121536143, max timestamp=1732121536157Num uniques 1018; 2024-11-20T16:52:16,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T16:52:16,190 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/1349769b0f1b4261b1121a45b1e4e867 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/1349769b0f1b4261b1121a45b1e4e867 2024-11-20T16:52:16,226 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/C of b683f7af58301e87dbf6f3d58dd983d2 into 1349769b0f1b4261b1121a45b1e4e867(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:16,226 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:16,227 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/C, priority=12, startTime=1732121535850; duration=0sec 2024-11-20T16:52:16,227 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:16,227 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:C 2024-11-20T16:52:16,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121596269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:16,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732121536257, max timestamp=1732121536293Num uniques 1027; 2024-11-20T16:52:16,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732121536257, max timestamp=1732121536293Num uniques 1027; 2024-11-20T16:52:16,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732121536257, max timestamp=1732121536293Num uniques 1027; 2024-11-20T16:52:16,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-20T16:52:16,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-20T16:52:16,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-20T16:52:16,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:16,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121596271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:16,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:16,382 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/c66fd4c50ea84713b69ed6e34a3c23e8 2024-11-20T16:52:16,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/035d7541dae144e9beba984b5da4d96d is 50, key is test_row_1/B:col10/1732121535655/Put/seqid=0 2024-11-20T16:52:16,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121596414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:16,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:16,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121596417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:16,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:16,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741853_1029 (size=9657) 2024-11-20T16:52:16,430 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/035d7541dae144e9beba984b5da4d96d 2024-11-20T16:52:16,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121596419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:16,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:16,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121596436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:16,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:16,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/e5d1dac3a663482f9128c2155ec4818a is 50, key is test_row_1/C:col10/1732121535655/Put/seqid=0 2024-11-20T16:52:16,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121596456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:16,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741854_1030 (size=9657) 2024-11-20T16:52:16,480 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/e5d1dac3a663482f9128c2155ec4818a 2024-11-20T16:52:16,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/c66fd4c50ea84713b69ed6e34a3c23e8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c66fd4c50ea84713b69ed6e34a3c23e8 2024-11-20T16:52:16,504 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c66fd4c50ea84713b69ed6e34a3c23e8, entries=100, sequenceid=104, filesize=9.4 K 2024-11-20T16:52:16,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/035d7541dae144e9beba984b5da4d96d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/035d7541dae144e9beba984b5da4d96d 2024-11-20T16:52:16,517 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/035d7541dae144e9beba984b5da4d96d, entries=100, sequenceid=104, filesize=9.4 K 2024-11-20T16:52:16,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/e5d1dac3a663482f9128c2155ec4818a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/e5d1dac3a663482f9128c2155ec4818a 2024-11-20T16:52:16,535 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/e5d1dac3a663482f9128c2155ec4818a, entries=100, sequenceid=104, filesize=9.4 K 2024-11-20T16:52:16,540 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for b683f7af58301e87dbf6f3d58dd983d2 in 597ms, sequenceid=104, compaction requested=true 2024-11-20T16:52:16,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:16,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:16,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-20T16:52:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-20T16:52:16,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T16:52:16,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1940 sec 2024-11-20T16:52:16,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:16,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=147.60 KB heapSize=413.77 KB 2024-11-20T16:52:16,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:16,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:16,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:16,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:16,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:16,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:16,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.2140 sec 2024-11-20T16:52:16,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121596566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:16,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/c5c8c044155d4445896085f27b6c60a8 is 50, key is test_row_0/A:col10/1732121536413/Put/seqid=0 2024-11-20T16:52:16,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:16,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121596575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121596577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:16,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741855_1031 (size=14541) 2024-11-20T16:52:16,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/c5c8c044155d4445896085f27b6c60a8 2024-11-20T16:52:16,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/f735c844a4c94c28944bc5694aac465c is 50, key is test_row_0/B:col10/1732121536413/Put/seqid=0 2024-11-20T16:52:16,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741856_1032 (size=12151) 2024-11-20T16:52:16,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/f735c844a4c94c28944bc5694aac465c 2024-11-20T16:52:16,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121596656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:16,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:16,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121596676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:16,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/9ba956502e874ce78ad5e53e0d1ec472 is 50, key is test_row_0/C:col10/1732121536413/Put/seqid=0 2024-11-20T16:52:16,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121596697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:16,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121596697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:16,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:16,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121596697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:16,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741857_1033 (size=12151) 2024-11-20T16:52:16,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/9ba956502e874ce78ad5e53e0d1ec472 2024-11-20T16:52:16,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/c5c8c044155d4445896085f27b6c60a8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c5c8c044155d4445896085f27b6c60a8 2024-11-20T16:52:16,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c5c8c044155d4445896085f27b6c60a8, entries=200, sequenceid=178, filesize=14.2 K 2024-11-20T16:52:16,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/f735c844a4c94c28944bc5694aac465c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/f735c844a4c94c28944bc5694aac465c 2024-11-20T16:52:16,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/f735c844a4c94c28944bc5694aac465c, entries=150, sequenceid=178, filesize=11.9 K 2024-11-20T16:52:16,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/9ba956502e874ce78ad5e53e0d1ec472 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/9ba956502e874ce78ad5e53e0d1ec472 2024-11-20T16:52:16,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/9ba956502e874ce78ad5e53e0d1ec472, entries=150, sequenceid=178, filesize=11.9 K 2024-11-20T16:52:16,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~413.72 KB/423648, currentSize=40.25 KB/41220 for b683f7af58301e87dbf6f3d58dd983d2 in 224ms, sequenceid=178, compaction requested=true 2024-11-20T16:52:16,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T16:52:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-20T16:52:16,774 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:52:16,774 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:52:16,778 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50643 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:52:16,778 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/A is initiating minor compaction (all files) 2024-11-20T16:52:16,778 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/A in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:16,779 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/f6d7d400f37940ffab44777e3fcbe617, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/7c1bbb8a57f045d98a0fb9978587233a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c66fd4c50ea84713b69ed6e34a3c23e8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c5c8c044155d4445896085f27b6c60a8] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=49.5 K 2024-11-20T16:52:16,779 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6d7d400f37940ffab44777e3fcbe617, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732121534445 2024-11-20T16:52:16,780 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:52:16,780 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/B is initiating minor compaction (all files) 2024-11-20T16:52:16,780 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/B in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:16,780 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/085a96f94036497ab14a56a1f592c524, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/f657202cb7894d158506a72134f0ba71, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/035d7541dae144e9beba984b5da4d96d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/f735c844a4c94c28944bc5694aac465c] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=44.8 K 2024-11-20T16:52:16,781 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c1bbb8a57f045d98a0fb9978587233a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732121534920 2024-11-20T16:52:16,781 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 085a96f94036497ab14a56a1f592c524, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732121534445 2024-11-20T16:52:16,782 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting c66fd4c50ea84713b69ed6e34a3c23e8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732121535631 2024-11-20T16:52:16,782 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting f657202cb7894d158506a72134f0ba71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732121534943 2024-11-20T16:52:16,784 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5c8c044155d4445896085f27b6c60a8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732121536402 2024-11-20T16:52:16,784 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 035d7541dae144e9beba984b5da4d96d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732121535631 2024-11-20T16:52:16,788 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting f735c844a4c94c28944bc5694aac465c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732121536406 2024-11-20T16:52:16,827 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#B#compaction#21 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:16,828 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/6650bad844384cfd8861d92084a933da is 50, key is test_row_0/B:col10/1732121536413/Put/seqid=0 2024-11-20T16:52:16,837 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#A#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:16,837 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/faa0434bcf5b401c9e24dad932194890 is 50, key is test_row_0/A:col10/1732121536413/Put/seqid=0 2024-11-20T16:52:16,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741858_1034 (size=12391) 2024-11-20T16:52:16,881 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/6650bad844384cfd8861d92084a933da as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/6650bad844384cfd8861d92084a933da 2024-11-20T16:52:16,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741859_1035 (size=12391) 2024-11-20T16:52:16,899 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/B of b683f7af58301e87dbf6f3d58dd983d2 into 6650bad844384cfd8861d92084a933da(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:16,899 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:16,899 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/B, priority=12, startTime=1732121536774; duration=0sec 2024-11-20T16:52:16,899 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:16,899 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:B 2024-11-20T16:52:16,900 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:16,905 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:16,905 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/C is initiating minor compaction (all files) 2024-11-20T16:52:16,905 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/C in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:16,906 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/1349769b0f1b4261b1121a45b1e4e867, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/e5d1dac3a663482f9128c2155ec4818a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/9ba956502e874ce78ad5e53e0d1ec472] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=33.2 K 2024-11-20T16:52:16,907 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 1349769b0f1b4261b1121a45b1e4e867, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732121534943 2024-11-20T16:52:16,908 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting e5d1dac3a663482f9128c2155ec4818a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732121535631 2024-11-20T16:52:16,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T16:52:16,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:16,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:16,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:16,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:16,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:16,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:16,914 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ba956502e874ce78ad5e53e0d1ec472, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732121536406 2024-11-20T16:52:16,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:16,918 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/faa0434bcf5b401c9e24dad932194890 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/faa0434bcf5b401c9e24dad932194890 2024-11-20T16:52:16,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/6d6bc545924e4e7db452e03bca86172f is 50, key is test_row_0/A:col10/1732121536910/Put/seqid=0 2024-11-20T16:52:16,943 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/A of b683f7af58301e87dbf6f3d58dd983d2 into faa0434bcf5b401c9e24dad932194890(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:16,943 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:16,943 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/A, priority=12, startTime=1732121536773; duration=0sec 2024-11-20T16:52:16,944 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:16,944 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:A 2024-11-20T16:52:16,947 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:16,948 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/ec87e3b23415451d807cd5cc88978f5a is 50, key is test_row_0/C:col10/1732121536413/Put/seqid=0 2024-11-20T16:52:16,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741860_1036 (size=12151) 2024-11-20T16:52:16,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/6d6bc545924e4e7db452e03bca86172f 2024-11-20T16:52:16,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732121536974, max timestamp=1732121536993Num uniques 859; 2024-11-20T16:52:16,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732121536974, max timestamp=1732121536993Num uniques 859; 2024-11-20T16:52:16,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T16:52:16,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T16:52:17,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732121536974, max timestamp=1732121536993Num uniques 859; 2024-11-20T16:52:17,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T16:52:17,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741861_1037 (size=12391) 2024-11-20T16:52:17,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/c77ef57cd47b4224b91c06a204ddcccf is 50, key is test_row_0/B:col10/1732121536910/Put/seqid=0 2024-11-20T16:52:17,025 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/ec87e3b23415451d807cd5cc88978f5a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/ec87e3b23415451d807cd5cc88978f5a 2024-11-20T16:52:17,045 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/C of b683f7af58301e87dbf6f3d58dd983d2 into ec87e3b23415451d807cd5cc88978f5a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:17,045 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:17,045 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/C, priority=13, startTime=1732121536774; duration=0sec 2024-11-20T16:52:17,045 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:17,045 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:C 2024-11-20T16:52:17,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741862_1038 (size=12151) 2024-11-20T16:52:17,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/c77ef57cd47b4224b91c06a204ddcccf 2024-11-20T16:52:17,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121597056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121597059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/68b9139eef2141258bd5f1c7d6c37d11 is 50, key is test_row_0/C:col10/1732121536910/Put/seqid=0 2024-11-20T16:52:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121597067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121597069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121597071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741863_1039 (size=12151) 2024-11-20T16:52:17,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/68b9139eef2141258bd5f1c7d6c37d11 2024-11-20T16:52:17,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/6d6bc545924e4e7db452e03bca86172f as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/6d6bc545924e4e7db452e03bca86172f 2024-11-20T16:52:17,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/6d6bc545924e4e7db452e03bca86172f, entries=150, sequenceid=189, filesize=11.9 K 2024-11-20T16:52:17,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/c77ef57cd47b4224b91c06a204ddcccf as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/c77ef57cd47b4224b91c06a204ddcccf 2024-11-20T16:52:17,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121597176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121597177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121597186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121597186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121597186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/c77ef57cd47b4224b91c06a204ddcccf, entries=150, sequenceid=189, filesize=11.9 K 2024-11-20T16:52:17,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/68b9139eef2141258bd5f1c7d6c37d11 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/68b9139eef2141258bd5f1c7d6c37d11 2024-11-20T16:52:17,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/68b9139eef2141258bd5f1c7d6c37d11, entries=150, sequenceid=189, filesize=11.9 K 2024-11-20T16:52:17,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for b683f7af58301e87dbf6f3d58dd983d2 in 292ms, sequenceid=189, compaction requested=false 2024-11-20T16:52:17,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:17,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-20T16:52:17,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:17,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:17,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:17,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:17,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:17,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:17,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/cf79a39f109d431c8c546c25d4c7b056 is 50, key is test_row_0/A:col10/1732121537387/Put/seqid=0 2024-11-20T16:52:17,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121597403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121597408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121597411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121597411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121597414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741864_1040 (size=14541) 2024-11-20T16:52:17,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/cf79a39f109d431c8c546c25d4c7b056 2024-11-20T16:52:17,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/c490e436e0d84ff9aee8d03612e27990 is 50, key is test_row_0/B:col10/1732121537387/Put/seqid=0 2024-11-20T16:52:17,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741865_1041 (size=12151) 2024-11-20T16:52:17,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/c490e436e0d84ff9aee8d03612e27990 2024-11-20T16:52:17,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/41470bad2bf843a1a648c1d0563a2bec is 50, key is test_row_0/C:col10/1732121537387/Put/seqid=0 2024-11-20T16:52:17,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T16:52:17,507 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:17,507 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:17,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-20T16:52:17,511 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:17,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T16:52:17,515 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:17,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:17,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121597526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121597527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121597527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121597527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121597527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741866_1042 (size=12151) 2024-11-20T16:52:17,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/41470bad2bf843a1a648c1d0563a2bec 2024-11-20T16:52:17,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/cf79a39f109d431c8c546c25d4c7b056 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/cf79a39f109d431c8c546c25d4c7b056 2024-11-20T16:52:17,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/cf79a39f109d431c8c546c25d4c7b056, entries=200, sequenceid=227, filesize=14.2 K 2024-11-20T16:52:17,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/c490e436e0d84ff9aee8d03612e27990 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/c490e436e0d84ff9aee8d03612e27990 2024-11-20T16:52:17,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/c490e436e0d84ff9aee8d03612e27990, entries=150, sequenceid=227, filesize=11.9 K 2024-11-20T16:52:17,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/41470bad2bf843a1a648c1d0563a2bec as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/41470bad2bf843a1a648c1d0563a2bec 2024-11-20T16:52:17,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T16:52:17,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/41470bad2bf843a1a648c1d0563a2bec, entries=150, sequenceid=227, filesize=11.9 K 2024-11-20T16:52:17,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T16:52:17,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:17,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:17,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:17,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:17,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:17,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for b683f7af58301e87dbf6f3d58dd983d2 in 282ms, sequenceid=227, compaction requested=true 2024-11-20T16:52:17,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:17,673 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:17,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:17,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:17,673 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:17,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:17,676 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:17,676 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/A is initiating minor compaction (all files) 2024-11-20T16:52:17,676 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/A in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:17,676 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/faa0434bcf5b401c9e24dad932194890, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/6d6bc545924e4e7db452e03bca86172f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/cf79a39f109d431c8c546c25d4c7b056] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=38.2 K 2024-11-20T16:52:17,676 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:17,676 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/B is initiating minor compaction (all files) 2024-11-20T16:52:17,677 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/B in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:17,677 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/6650bad844384cfd8861d92084a933da, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/c77ef57cd47b4224b91c06a204ddcccf, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/c490e436e0d84ff9aee8d03612e27990] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=35.8 K 2024-11-20T16:52:17,677 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting faa0434bcf5b401c9e24dad932194890, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732121536406 2024-11-20T16:52:17,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:17,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:17,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:17,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:17,678 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d6bc545924e4e7db452e03bca86172f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732121536563 2024-11-20T16:52:17,678 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 6650bad844384cfd8861d92084a933da, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732121536406 2024-11-20T16:52:17,679 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf79a39f109d431c8c546c25d4c7b056, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732121537063 2024-11-20T16:52:17,679 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting c77ef57cd47b4224b91c06a204ddcccf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732121536563 2024-11-20T16:52:17,680 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting c490e436e0d84ff9aee8d03612e27990, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732121537063 2024-11-20T16:52:17,702 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#B#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:17,703 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/fbb00160643b419abecafcbf30f3ad18 is 50, key is test_row_0/B:col10/1732121537387/Put/seqid=0 2024-11-20T16:52:17,707 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#A#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:17,708 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/08121d6a9b6740e18934d9b67a57273a is 50, key is test_row_0/A:col10/1732121537387/Put/seqid=0 2024-11-20T16:52:17,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741868_1044 (size=12493) 2024-11-20T16:52:17,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741867_1043 (size=12493) 2024-11-20T16:52:17,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:17,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T16:52:17,769 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/08121d6a9b6740e18934d9b67a57273a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/08121d6a9b6740e18934d9b67a57273a 2024-11-20T16:52:17,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:17,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:17,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:17,786 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/A of b683f7af58301e87dbf6f3d58dd983d2 into 08121d6a9b6740e18934d9b67a57273a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:17,786 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:17,786 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/A, priority=13, startTime=1732121537673; duration=0sec 2024-11-20T16:52:17,786 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:17,786 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:A 2024-11-20T16:52:17,786 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:17,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/3166c3428aea4b8bb6b6c2eb72d385a4 is 50, key is test_row_0/A:col10/1732121537764/Put/seqid=0 2024-11-20T16:52:17,789 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:17,792 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/C is initiating minor compaction (all files) 2024-11-20T16:52:17,792 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/C in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:17,793 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/ec87e3b23415451d807cd5cc88978f5a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/68b9139eef2141258bd5f1c7d6c37d11, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/41470bad2bf843a1a648c1d0563a2bec] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=35.8 K 2024-11-20T16:52:17,796 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec87e3b23415451d807cd5cc88978f5a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732121536406 2024-11-20T16:52:17,796 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68b9139eef2141258bd5f1c7d6c37d11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732121536563 2024-11-20T16:52:17,801 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41470bad2bf843a1a648c1d0563a2bec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732121537063 2024-11-20T16:52:17,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741869_1045 (size=12151) 2024-11-20T16:52:17,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/3166c3428aea4b8bb6b6c2eb72d385a4 2024-11-20T16:52:17,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T16:52:17,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:17,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:17,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:17,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:17,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:17,835 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#C#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T16:52:17,837 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/efd4b4857742492fb06019f49c2a650c is 50, key is test_row_0/C:col10/1732121537387/Put/seqid=0 2024-11-20T16:52:17,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/b530438e9a47433f8cdad33a0281015b is 50, key is test_row_0/B:col10/1732121537764/Put/seqid=0 2024-11-20T16:52:17,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121597827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121597832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121597836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121597840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121597841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741870_1046 (size=12151) 2024-11-20T16:52:17,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/b530438e9a47433f8cdad33a0281015b 2024-11-20T16:52:17,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741871_1047 (size=12493) 2024-11-20T16:52:17,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/db91ee01589843fea5e7bcf11efd4efd is 50, key is test_row_0/C:col10/1732121537764/Put/seqid=0 2024-11-20T16:52:17,901 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/efd4b4857742492fb06019f49c2a650c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/efd4b4857742492fb06019f49c2a650c 2024-11-20T16:52:17,914 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/C of b683f7af58301e87dbf6f3d58dd983d2 into efd4b4857742492fb06019f49c2a650c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:17,914 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:17,914 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/C, priority=13, startTime=1732121537677; duration=0sec 2024-11-20T16:52:17,914 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:17,914 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:C 2024-11-20T16:52:17,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741872_1048 (size=12151) 2024-11-20T16:52:17,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/db91ee01589843fea5e7bcf11efd4efd 2024-11-20T16:52:17,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121597946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121597946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/3166c3428aea4b8bb6b6c2eb72d385a4 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/3166c3428aea4b8bb6b6c2eb72d385a4 2024-11-20T16:52:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/3166c3428aea4b8bb6b6c2eb72d385a4, entries=150, sequenceid=240, filesize=11.9 K 2024-11-20T16:52:17,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121597956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/b530438e9a47433f8cdad33a0281015b as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b530438e9a47433f8cdad33a0281015b 2024-11-20T16:52:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121597956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121597957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:17,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:17,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b530438e9a47433f8cdad33a0281015b, entries=150, sequenceid=240, filesize=11.9 K 2024-11-20T16:52:17,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/db91ee01589843fea5e7bcf11efd4efd as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/db91ee01589843fea5e7bcf11efd4efd 2024-11-20T16:52:17,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/db91ee01589843fea5e7bcf11efd4efd, entries=150, sequenceid=240, filesize=11.9 K 2024-11-20T16:52:17,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T16:52:17,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:17,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for b683f7af58301e87dbf6f3d58dd983d2 in 222ms, sequenceid=240, compaction requested=false 2024-11-20T16:52:17,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:17,988 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T16:52:17,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:17,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:17,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:17,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:17,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:17,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:18,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/db0c2d94e98b442386cd9330f1480f24 is 50, key is test_row_0/A:col10/1732121537835/Put/seqid=0 2024-11-20T16:52:18,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741873_1049 (size=12301) 2024-11-20T16:52:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T16:52:18,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:18,165 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/fbb00160643b419abecafcbf30f3ad18 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/fbb00160643b419abecafcbf30f3ad18 2024-11-20T16:52:18,180 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/B of b683f7af58301e87dbf6f3d58dd983d2 into fbb00160643b419abecafcbf30f3ad18(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:18,180 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:18,180 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/B, priority=13, startTime=1732121537673; duration=0sec 2024-11-20T16:52:18,181 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:18,181 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:B 2024-11-20T16:52:18,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121598178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121598184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121598181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121598193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121598196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121598307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121598307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121598308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121598308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121598308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,445 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/db0c2d94e98b442386cd9330f1480f24 2024-11-20T16:52:18,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/b3d843e8389846aa8202c003d8a9d0f9 is 50, key is test_row_0/B:col10/1732121537835/Put/seqid=0 2024-11-20T16:52:18,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741874_1050 (size=12301) 2024-11-20T16:52:18,492 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/b3d843e8389846aa8202c003d8a9d0f9 2024-11-20T16:52:18,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/73ba34ab14aa4e169af97189804c40e0 is 50, key is test_row_0/C:col10/1732121537835/Put/seqid=0 2024-11-20T16:52:18,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121598516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121598516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741875_1051 (size=12301) 2024-11-20T16:52:18,526 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/73ba34ab14aa4e169af97189804c40e0 2024-11-20T16:52:18,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121598516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121598517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121598517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/db0c2d94e98b442386cd9330f1480f24 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/db0c2d94e98b442386cd9330f1480f24 2024-11-20T16:52:18,555 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/db0c2d94e98b442386cd9330f1480f24, entries=150, sequenceid=262, filesize=12.0 K 2024-11-20T16:52:18,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/b3d843e8389846aa8202c003d8a9d0f9 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b3d843e8389846aa8202c003d8a9d0f9 2024-11-20T16:52:18,571 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b3d843e8389846aa8202c003d8a9d0f9, entries=150, sequenceid=262, filesize=12.0 K 2024-11-20T16:52:18,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/73ba34ab14aa4e169af97189804c40e0 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/73ba34ab14aa4e169af97189804c40e0 2024-11-20T16:52:18,593 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/73ba34ab14aa4e169af97189804c40e0, entries=150, sequenceid=262, filesize=12.0 K 2024-11-20T16:52:18,596 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for b683f7af58301e87dbf6f3d58dd983d2 in 608ms, sequenceid=262, compaction requested=true 2024-11-20T16:52:18,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:18,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:18,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-20T16:52:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-20T16:52:18,603 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-20T16:52:18,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0840 sec 2024-11-20T16:52:18,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.0970 sec 2024-11-20T16:52:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T16:52:18,656 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:18,659 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-20T16:52:18,663 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T16:52:18,665 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:18,665 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T16:52:18,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T16:52:18,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:18,821 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T16:52:18,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:18,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:18,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:18,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:18,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:18,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:18,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/1e06aaf2c04b42a296577ca18d801c0a is 50, key is test_row_0/A:col10/1732121538190/Put/seqid=0 2024-11-20T16:52:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:18,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:18,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741876_1052 (size=12301) 2024-11-20T16:52:18,867 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/1e06aaf2c04b42a296577ca18d801c0a 2024-11-20T16:52:18,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/7c683e33b1a84ba1ab5151b72928d894 is 50, key is test_row_0/B:col10/1732121538190/Put/seqid=0 2024-11-20T16:52:18,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121598887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121598892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121598892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121598896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121598896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:18,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741877_1053 (size=12301) 2024-11-20T16:52:18,927 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/7c683e33b1a84ba1ab5151b72928d894 2024-11-20T16:52:18,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/e976eee0c8fb4e89befc137aecd6aa9a is 50, key is test_row_0/C:col10/1732121538190/Put/seqid=0 2024-11-20T16:52:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T16:52:18,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741878_1054 (size=12301) 2024-11-20T16:52:18,995 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/e976eee0c8fb4e89befc137aecd6aa9a 2024-11-20T16:52:19,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/1e06aaf2c04b42a296577ca18d801c0a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/1e06aaf2c04b42a296577ca18d801c0a 2024-11-20T16:52:19,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121599006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121599006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121599007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,016 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/1e06aaf2c04b42a296577ca18d801c0a, entries=150, sequenceid=276, filesize=12.0 K 2024-11-20T16:52:19,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/7c683e33b1a84ba1ab5151b72928d894 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/7c683e33b1a84ba1ab5151b72928d894 2024-11-20T16:52:19,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121599016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121599016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,029 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/7c683e33b1a84ba1ab5151b72928d894, entries=150, sequenceid=276, filesize=12.0 K 2024-11-20T16:52:19,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/e976eee0c8fb4e89befc137aecd6aa9a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/e976eee0c8fb4e89befc137aecd6aa9a 2024-11-20T16:52:19,048 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/e976eee0c8fb4e89befc137aecd6aa9a, entries=150, sequenceid=276, filesize=12.0 K 2024-11-20T16:52:19,051 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for b683f7af58301e87dbf6f3d58dd983d2 in 231ms, sequenceid=276, compaction requested=true 2024-11-20T16:52:19,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:19,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:19,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-20T16:52:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-20T16:52:19,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-20T16:52:19,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 388 msec 2024-11-20T16:52:19,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 400 msec 2024-11-20T16:52:19,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-20T16:52:19,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:19,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:19,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:19,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:19,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:19,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:19,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/797c708c94e44d69860f7772665ae253 is 50, key is test_row_0/A:col10/1732121538855/Put/seqid=0 2024-11-20T16:52:19,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741879_1055 (size=14741) 2024-11-20T16:52:19,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/797c708c94e44d69860f7772665ae253 2024-11-20T16:52:19,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121599244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/d6cb531401c0459496105b5974a6fe1c is 50, key is test_row_0/B:col10/1732121538855/Put/seqid=0 2024-11-20T16:52:19,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121599247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121599256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121599269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121599269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T16:52:19,287 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:19,288 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-20T16:52:19,293 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:19,295 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:19,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T16:52:19,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741880_1056 (size=12301) 2024-11-20T16:52:19,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/d6cb531401c0459496105b5974a6fe1c 2024-11-20T16:52:19,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/27e57b5423e746a2ae7e66fefe2e5edd is 50, key is test_row_0/C:col10/1732121538855/Put/seqid=0 2024-11-20T16:52:19,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741881_1057 (size=12301) 2024-11-20T16:52:19,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121599377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121599377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121599381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121599386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121599387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T16:52:19,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T16:52:19,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:19,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:19,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:19,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:19,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:19,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121599596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T16:52:19,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:19,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:19,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121599596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121599596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121599597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121599597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T16:52:19,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T16:52:19,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:19,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:19,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:19,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:19,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:19,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/27e57b5423e746a2ae7e66fefe2e5edd 2024-11-20T16:52:19,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/797c708c94e44d69860f7772665ae253 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/797c708c94e44d69860f7772665ae253 2024-11-20T16:52:19,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/797c708c94e44d69860f7772665ae253, entries=200, sequenceid=297, filesize=14.4 K 2024-11-20T16:52:19,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/d6cb531401c0459496105b5974a6fe1c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/d6cb531401c0459496105b5974a6fe1c 2024-11-20T16:52:19,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/d6cb531401c0459496105b5974a6fe1c, entries=150, sequenceid=297, filesize=12.0 K 2024-11-20T16:52:19,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/27e57b5423e746a2ae7e66fefe2e5edd as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/27e57b5423e746a2ae7e66fefe2e5edd 2024-11-20T16:52:19,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/27e57b5423e746a2ae7e66fefe2e5edd, entries=150, sequenceid=297, filesize=12.0 K 2024-11-20T16:52:19,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for b683f7af58301e87dbf6f3d58dd983d2 in 589ms, sequenceid=297, compaction requested=true 2024-11-20T16:52:19,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:19,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:19,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:19,812 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T16:52:19,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:19,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:19,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:19,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T16:52:19,812 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T16:52:19,822 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 63987 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T16:52:19,823 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/A is initiating minor compaction (all files) 2024-11-20T16:52:19,823 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61547 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T16:52:19,823 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/B is initiating minor compaction (all files) 2024-11-20T16:52:19,823 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/A in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:19,823 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/B in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:19,823 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/08121d6a9b6740e18934d9b67a57273a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/3166c3428aea4b8bb6b6c2eb72d385a4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/db0c2d94e98b442386cd9330f1480f24, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/1e06aaf2c04b42a296577ca18d801c0a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/797c708c94e44d69860f7772665ae253] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=62.5 K 2024-11-20T16:52:19,823 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/fbb00160643b419abecafcbf30f3ad18, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b530438e9a47433f8cdad33a0281015b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b3d843e8389846aa8202c003d8a9d0f9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/7c683e33b1a84ba1ab5151b72928d894, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/d6cb531401c0459496105b5974a6fe1c] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=60.1 K 2024-11-20T16:52:19,824 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08121d6a9b6740e18934d9b67a57273a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732121537063 2024-11-20T16:52:19,824 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting fbb00160643b419abecafcbf30f3ad18, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732121537063 2024-11-20T16:52:19,824 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3166c3428aea4b8bb6b6c2eb72d385a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732121537410 2024-11-20T16:52:19,824 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting b530438e9a47433f8cdad33a0281015b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732121537410 2024-11-20T16:52:19,825 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting b3d843e8389846aa8202c003d8a9d0f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732121537831 2024-11-20T16:52:19,825 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting db0c2d94e98b442386cd9330f1480f24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732121537831 2024-11-20T16:52:19,825 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c683e33b1a84ba1ab5151b72928d894, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732121538181 2024-11-20T16:52:19,825 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e06aaf2c04b42a296577ca18d801c0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732121538181 2024-11-20T16:52:19,826 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting d6cb531401c0459496105b5974a6fe1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732121538855 2024-11-20T16:52:19,826 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 797c708c94e44d69860f7772665ae253, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732121538855 2024-11-20T16:52:19,849 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#B#compaction#45 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:19,850 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/4c3ecc43eb564f2cacedb4de4e2ebd85 is 50, key is test_row_0/B:col10/1732121538855/Put/seqid=0 2024-11-20T16:52:19,854 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#A#compaction#46 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:19,855 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/14c67f17957140ed9f3d00bbb0cc54ea is 50, key is test_row_0/A:col10/1732121538855/Put/seqid=0 2024-11-20T16:52:19,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741882_1058 (size=12813) 2024-11-20T16:52:19,888 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/14c67f17957140ed9f3d00bbb0cc54ea as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/14c67f17957140ed9f3d00bbb0cc54ea 2024-11-20T16:52:19,899 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/A of b683f7af58301e87dbf6f3d58dd983d2 into 14c67f17957140ed9f3d00bbb0cc54ea(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:19,899 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:19,899 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/A, priority=11, startTime=1732121539811; duration=0sec 2024-11-20T16:52:19,899 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:19,899 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:A 2024-11-20T16:52:19,899 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T16:52:19,905 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61547 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T16:52:19,905 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/C is initiating minor compaction (all files) 2024-11-20T16:52:19,905 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/C in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:19,905 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/efd4b4857742492fb06019f49c2a650c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/db91ee01589843fea5e7bcf11efd4efd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/73ba34ab14aa4e169af97189804c40e0, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/e976eee0c8fb4e89befc137aecd6aa9a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/27e57b5423e746a2ae7e66fefe2e5edd] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=60.1 K 2024-11-20T16:52:19,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741883_1059 (size=12813) 2024-11-20T16:52:19,908 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting efd4b4857742492fb06019f49c2a650c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732121537063 2024-11-20T16:52:19,908 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting db91ee01589843fea5e7bcf11efd4efd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732121537410 2024-11-20T16:52:19,909 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73ba34ab14aa4e169af97189804c40e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732121537831 2024-11-20T16:52:19,910 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting e976eee0c8fb4e89befc137aecd6aa9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732121538181 2024-11-20T16:52:19,911 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27e57b5423e746a2ae7e66fefe2e5edd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732121538855 2024-11-20T16:52:19,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T16:52:19,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:19,913 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T16:52:19,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:19,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:19,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:19,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:19,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:19,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:19,918 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/4c3ecc43eb564f2cacedb4de4e2ebd85 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/4c3ecc43eb564f2cacedb4de4e2ebd85 2024-11-20T16:52:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:19,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:19,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/c687973d974642bbbfd5aa0a8c379d77 is 50, key is test_row_0/A:col10/1732121539240/Put/seqid=0 2024-11-20T16:52:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T16:52:19,931 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/B of b683f7af58301e87dbf6f3d58dd983d2 into 4c3ecc43eb564f2cacedb4de4e2ebd85(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:19,931 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:19,931 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/B, priority=11, startTime=1732121539812; duration=0sec 2024-11-20T16:52:19,931 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:19,932 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:B 2024-11-20T16:52:19,942 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#C#compaction#48 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:19,943 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/8ed2bd49e45a4fd0ac95029769bdff55 is 50, key is test_row_0/C:col10/1732121538855/Put/seqid=0 2024-11-20T16:52:19,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121599973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732121539968, max timestamp=1732121539981Num uniques 859; 2024-11-20T16:52:19,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:19,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732121539968, max timestamp=1732121539981Num uniques 859; 2024-11-20T16:52:19,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121599973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:19,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T16:52:19,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732121539968, max timestamp=1732121539981Num uniques 859; 2024-11-20T16:52:19,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T16:52:19,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T16:52:19,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:19,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:19,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741884_1060 (size=12301) 2024-11-20T16:52:19,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741885_1061 (size=12813) 2024-11-20T16:52:20,010 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/8ed2bd49e45a4fd0ac95029769bdff55 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/8ed2bd49e45a4fd0ac95029769bdff55 2024-11-20T16:52:20,022 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/C of b683f7af58301e87dbf6f3d58dd983d2 into 8ed2bd49e45a4fd0ac95029769bdff55(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:20,022 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:20,022 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/C, priority=11, startTime=1732121539812; duration=0sec 2024-11-20T16:52:20,023 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:20,023 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:C 2024-11-20T16:52:20,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121600042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:20,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:20,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121600044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:20,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:20,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:20,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121600046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:20,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121600097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:20,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121600098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:20,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121600157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:20,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121600157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:20,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:20,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121600166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:20,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121600306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:20,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:20,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121600307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:20,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:20,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121600366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:20,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121600366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:20,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121600386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:20,390 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/c687973d974642bbbfd5aa0a8c379d77 2024-11-20T16:52:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/f0239be5e2064f4ba3652cedb3d1b573 is 50, key is test_row_0/B:col10/1732121539240/Put/seqid=0 2024-11-20T16:52:20,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741886_1062 (size=12301) 2024-11-20T16:52:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T16:52:20,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121600616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:20,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121600616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:20,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121600676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121600676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:20,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121600706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:20,834 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/f0239be5e2064f4ba3652cedb3d1b573 2024-11-20T16:52:20,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/3583e366b1d544a3b6fbf220e5c624e1 is 50, key is test_row_0/C:col10/1732121539240/Put/seqid=0 2024-11-20T16:52:20,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741887_1063 (size=12301) 2024-11-20T16:52:20,868 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/3583e366b1d544a3b6fbf220e5c624e1 2024-11-20T16:52:20,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/c687973d974642bbbfd5aa0a8c379d77 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c687973d974642bbbfd5aa0a8c379d77 2024-11-20T16:52:20,886 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c687973d974642bbbfd5aa0a8c379d77, entries=150, sequenceid=309, filesize=12.0 K 2024-11-20T16:52:20,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/f0239be5e2064f4ba3652cedb3d1b573 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/f0239be5e2064f4ba3652cedb3d1b573 2024-11-20T16:52:20,898 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/f0239be5e2064f4ba3652cedb3d1b573, entries=150, sequenceid=309, filesize=12.0 K 2024-11-20T16:52:20,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/3583e366b1d544a3b6fbf220e5c624e1 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3583e366b1d544a3b6fbf220e5c624e1 2024-11-20T16:52:20,908 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3583e366b1d544a3b6fbf220e5c624e1, entries=150, sequenceid=309, filesize=12.0 K 2024-11-20T16:52:20,910 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for b683f7af58301e87dbf6f3d58dd983d2 in 996ms, sequenceid=309, compaction requested=false 2024-11-20T16:52:20,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:20,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:20,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-20T16:52:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-20T16:52:20,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-20T16:52:20,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6170 sec 2024-11-20T16:52:20,919 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.6280 sec 2024-11-20T16:52:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:21,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-20T16:52:21,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:21,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:21,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:21,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:21,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:21,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:21,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/aa89f47dffae45a3b2df91e29e263f43 is 50, key is test_row_0/A:col10/1732121541126/Put/seqid=0 2024-11-20T16:52:21,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741888_1064 (size=12301) 2024-11-20T16:52:21,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121601177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:21,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:21,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:21,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121601184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:21,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:21,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121601196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121601196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:21,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:21,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:21,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121601216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:21,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121601296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:21,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121601296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T16:52:21,446 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:21,446 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-20T16:52:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T16:52:21,448 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:21,449 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:21,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:21,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:21,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121601507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:21,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:21,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121601507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:21,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T16:52:21,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/aa89f47dffae45a3b2df91e29e263f43 2024-11-20T16:52:21,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/68f75ce3f5fc45cea629d5bdd49fc791 is 50, key is test_row_0/B:col10/1732121541126/Put/seqid=0 2024-11-20T16:52:21,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741889_1065 (size=12301) 2024-11-20T16:52:21,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/68f75ce3f5fc45cea629d5bdd49fc791 2024-11-20T16:52:21,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T16:52:21,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:21,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:21,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:21,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:21,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:21,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/42d9585b79e949f7b42dd0f04fdd1d0b is 50, key is test_row_0/C:col10/1732121541126/Put/seqid=0 2024-11-20T16:52:21,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741890_1066 (size=12301) 2024-11-20T16:52:21,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T16:52:21,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:21,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:21,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:21,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:21,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T16:52:21,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121601816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:21,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121601826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:21,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T16:52:21,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:21,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:21,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:21,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:21,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:22,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/42d9585b79e949f7b42dd0f04fdd1d0b 2024-11-20T16:52:22,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/aa89f47dffae45a3b2df91e29e263f43 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/aa89f47dffae45a3b2df91e29e263f43 2024-11-20T16:52:22,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/aa89f47dffae45a3b2df91e29e263f43, entries=150, sequenceid=346, filesize=12.0 K 2024-11-20T16:52:22,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/68f75ce3f5fc45cea629d5bdd49fc791 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/68f75ce3f5fc45cea629d5bdd49fc791 2024-11-20T16:52:22,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/68f75ce3f5fc45cea629d5bdd49fc791, entries=150, sequenceid=346, filesize=12.0 K 2024-11-20T16:52:22,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T16:52:22,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:22,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:22,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:22,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:22,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:22,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/42d9585b79e949f7b42dd0f04fdd1d0b as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/42d9585b79e949f7b42dd0f04fdd1d0b 2024-11-20T16:52:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:22,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/42d9585b79e949f7b42dd0f04fdd1d0b, entries=150, sequenceid=346, filesize=12.0 K 2024-11-20T16:52:22,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for b683f7af58301e87dbf6f3d58dd983d2 in 954ms, sequenceid=346, compaction requested=true 2024-11-20T16:52:22,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:22,083 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:22,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:22,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:22,084 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:22,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:22,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:22,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:22,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:22,084 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:22,084 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/A is initiating minor compaction (all files) 2024-11-20T16:52:22,084 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/A in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:22,085 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/14c67f17957140ed9f3d00bbb0cc54ea, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c687973d974642bbbfd5aa0a8c379d77, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/aa89f47dffae45a3b2df91e29e263f43] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=36.5 K 2024-11-20T16:52:22,086 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14c67f17957140ed9f3d00bbb0cc54ea, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732121538855 2024-11-20T16:52:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T16:52:22,086 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:22,086 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting c687973d974642bbbfd5aa0a8c379d77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732121539239 2024-11-20T16:52:22,086 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/B is initiating minor compaction (all files) 2024-11-20T16:52:22,086 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/B in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:22,087 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/4c3ecc43eb564f2cacedb4de4e2ebd85, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/f0239be5e2064f4ba3652cedb3d1b573, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/68f75ce3f5fc45cea629d5bdd49fc791] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=36.5 K 2024-11-20T16:52:22,087 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa89f47dffae45a3b2df91e29e263f43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732121540032 2024-11-20T16:52:22,087 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c3ecc43eb564f2cacedb4de4e2ebd85, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732121538855 2024-11-20T16:52:22,088 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting f0239be5e2064f4ba3652cedb3d1b573, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732121539239 2024-11-20T16:52:22,088 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 68f75ce3f5fc45cea629d5bdd49fc791, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732121540032 2024-11-20T16:52:22,105 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#B#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:22,105 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/82326f9270a145fe9bc7ed425180b639 is 50, key is test_row_0/B:col10/1732121541126/Put/seqid=0 2024-11-20T16:52:22,118 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#A#compaction#55 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:22,119 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/e3695985298e4e7a8f8c6097650e3d67 is 50, key is test_row_0/A:col10/1732121541126/Put/seqid=0 2024-11-20T16:52:22,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741891_1067 (size=12915) 2024-11-20T16:52:22,155 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/82326f9270a145fe9bc7ed425180b639 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/82326f9270a145fe9bc7ed425180b639 2024-11-20T16:52:22,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741892_1068 (size=12915) 2024-11-20T16:52:22,166 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/B of b683f7af58301e87dbf6f3d58dd983d2 into 82326f9270a145fe9bc7ed425180b639(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:22,166 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:22,170 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/B, priority=13, startTime=1732121542083; duration=0sec 2024-11-20T16:52:22,170 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:22,171 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:B 2024-11-20T16:52:22,171 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:22,173 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:22,174 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/C is initiating minor compaction (all files) 2024-11-20T16:52:22,174 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/C in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:22,174 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/8ed2bd49e45a4fd0ac95029769bdff55, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3583e366b1d544a3b6fbf220e5c624e1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/42d9585b79e949f7b42dd0f04fdd1d0b] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=36.5 K 2024-11-20T16:52:22,175 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ed2bd49e45a4fd0ac95029769bdff55, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732121538855 2024-11-20T16:52:22,175 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 3583e366b1d544a3b6fbf220e5c624e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732121539239 2024-11-20T16:52:22,176 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 42d9585b79e949f7b42dd0f04fdd1d0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732121540032 2024-11-20T16:52:22,201 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#C#compaction#56 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:22,202 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/52b669afb9684fc5b8a12921697642b9 is 50, key is test_row_0/C:col10/1732121541126/Put/seqid=0 2024-11-20T16:52:22,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741893_1069 (size=12915) 2024-11-20T16:52:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:22,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T16:52:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:22,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T16:52:22,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:22,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:22,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:22,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:22,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:22,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/c4bd4b602a4a4d48ae6bf39f289060c2 is 50, key is test_row_0/A:col10/1732121541150/Put/seqid=0 2024-11-20T16:52:22,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741894_1070 (size=12301) 2024-11-20T16:52:22,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121602269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:22,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121602275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:22,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:22,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121602277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:22,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:22,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121602327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:22,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121602337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:22,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T16:52:22,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:22,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:22,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:22,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:22,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:22,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121602387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:22,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:22,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121602387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:22,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:22,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121602396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:22,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T16:52:22,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:22,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:22,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:22,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:22,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:22,566 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/e3695985298e4e7a8f8c6097650e3d67 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/e3695985298e4e7a8f8c6097650e3d67 2024-11-20T16:52:22,574 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/A of b683f7af58301e87dbf6f3d58dd983d2 into e3695985298e4e7a8f8c6097650e3d67(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:22,574 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:22,574 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/A, priority=13, startTime=1732121542083; duration=0sec 2024-11-20T16:52:22,574 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:22,574 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:A 2024-11-20T16:52:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T16:52:22,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121602597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121602597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:22,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121602606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:22,616 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/52b669afb9684fc5b8a12921697642b9 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/52b669afb9684fc5b8a12921697642b9 2024-11-20T16:52:22,628 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/C of b683f7af58301e87dbf6f3d58dd983d2 into 52b669afb9684fc5b8a12921697642b9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:22,628 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:22,628 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/C, priority=13, startTime=1732121542084; duration=0sec 2024-11-20T16:52:22,629 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:22,629 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:C 2024-11-20T16:52:22,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/c4bd4b602a4a4d48ae6bf39f289060c2 2024-11-20T16:52:22,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/ffda9d956a4340b6b4ff3bfcde39660a is 50, key is test_row_0/B:col10/1732121541150/Put/seqid=0 2024-11-20T16:52:22,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T16:52:22,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:22,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:22,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:22,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:22,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:22,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741895_1071 (size=12301) 2024-11-20T16:52:22,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/ffda9d956a4340b6b4ff3bfcde39660a 2024-11-20T16:52:22,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/543eaf97df9c494d99f89cac9118af77 is 50, key is test_row_0/C:col10/1732121541150/Put/seqid=0 2024-11-20T16:52:22,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741896_1072 (size=12301) 2024-11-20T16:52:22,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/543eaf97df9c494d99f89cac9118af77 2024-11-20T16:52:22,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/c4bd4b602a4a4d48ae6bf39f289060c2 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c4bd4b602a4a4d48ae6bf39f289060c2 2024-11-20T16:52:22,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c4bd4b602a4a4d48ae6bf39f289060c2, entries=150, sequenceid=359, filesize=12.0 K 2024-11-20T16:52:22,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/ffda9d956a4340b6b4ff3bfcde39660a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/ffda9d956a4340b6b4ff3bfcde39660a 2024-11-20T16:52:22,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/ffda9d956a4340b6b4ff3bfcde39660a, entries=150, sequenceid=359, filesize=12.0 K 2024-11-20T16:52:22,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/543eaf97df9c494d99f89cac9118af77 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/543eaf97df9c494d99f89cac9118af77 2024-11-20T16:52:22,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/543eaf97df9c494d99f89cac9118af77, entries=150, sequenceid=359, filesize=12.0 K 2024-11-20T16:52:22,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b683f7af58301e87dbf6f3d58dd983d2 in 568ms, sequenceid=359, compaction requested=false 2024-11-20T16:52:22,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:22,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T16:52:22,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:22,844 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T16:52:22,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:22,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:22,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:22,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:22,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:22,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:22,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/4a895c1b259f4c1e8df7f8f4605496be is 50, key is test_row_0/A:col10/1732121542273/Put/seqid=0 2024-11-20T16:52:22,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741897_1073 (size=12301) 2024-11-20T16:52:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:22,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:22,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121602926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:22,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121602933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:22,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121602935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:23,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121603036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:23,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121603046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:23,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121603047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:23,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:23,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121603246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:23,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121603256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:23,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121603256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:23,265 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/4a895c1b259f4c1e8df7f8f4605496be 2024-11-20T16:52:23,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/58ea3b9d267b4bef919a46fe630165dc is 50, key is test_row_0/B:col10/1732121542273/Put/seqid=0 2024-11-20T16:52:23,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741898_1074 (size=12301) 2024-11-20T16:52:23,290 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/58ea3b9d267b4bef919a46fe630165dc 2024-11-20T16:52:23,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/3f330077b8eb43c680b71e24c0131c21 is 50, key is test_row_0/C:col10/1732121542273/Put/seqid=0 2024-11-20T16:52:23,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741899_1075 (size=12301) 2024-11-20T16:52:23,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121603346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:23,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121603346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:23,351 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in b683f7af58301e87dbf6f3d58dd983d2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:52:23,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:23,359 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in b683f7af58301e87dbf6f3d58dd983d2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:52:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:23,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121603566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:23,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121603566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:23,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121603567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:23,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:23,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T16:52:23,717 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/3f330077b8eb43c680b71e24c0131c21 2024-11-20T16:52:23,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/4a895c1b259f4c1e8df7f8f4605496be as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/4a895c1b259f4c1e8df7f8f4605496be 2024-11-20T16:52:23,732 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/4a895c1b259f4c1e8df7f8f4605496be, entries=150, sequenceid=381, filesize=12.0 K 2024-11-20T16:52:23,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/58ea3b9d267b4bef919a46fe630165dc as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/58ea3b9d267b4bef919a46fe630165dc 2024-11-20T16:52:23,743 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/58ea3b9d267b4bef919a46fe630165dc, entries=150, sequenceid=381, filesize=12.0 K 2024-11-20T16:52:23,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/3f330077b8eb43c680b71e24c0131c21 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3f330077b8eb43c680b71e24c0131c21 2024-11-20T16:52:23,751 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3f330077b8eb43c680b71e24c0131c21, entries=150, sequenceid=381, filesize=12.0 K 2024-11-20T16:52:23,752 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b683f7af58301e87dbf6f3d58dd983d2 in 908ms, sequenceid=381, compaction requested=true 2024-11-20T16:52:23,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:23,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:23,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-20T16:52:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-20T16:52:23,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-20T16:52:23,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3050 sec 2024-11-20T16:52:23,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.3120 sec 2024-11-20T16:52:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:24,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T16:52:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:24,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/70accd13a1e8437f85e272bd7d227a8d is 50, key is test_row_0/A:col10/1732121544078/Put/seqid=0 2024-11-20T16:52:24,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741900_1076 (size=14741) 2024-11-20T16:52:24,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/70accd13a1e8437f85e272bd7d227a8d 2024-11-20T16:52:24,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/b6664a410be248bc956b60c73ac36d49 is 50, key is test_row_0/B:col10/1732121544078/Put/seqid=0 2024-11-20T16:52:24,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741901_1077 (size=12301) 2024-11-20T16:52:24,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121604122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:24,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:24,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121604122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:24,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:24,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121604123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:24,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/b6664a410be248bc956b60c73ac36d49 2024-11-20T16:52:24,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/4af45641b61a43a98d71e25bebc78903 is 50, key is test_row_0/C:col10/1732121544078/Put/seqid=0 2024-11-20T16:52:24,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741902_1078 (size=12301) 2024-11-20T16:52:24,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/4af45641b61a43a98d71e25bebc78903 2024-11-20T16:52:24,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/70accd13a1e8437f85e272bd7d227a8d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/70accd13a1e8437f85e272bd7d227a8d 2024-11-20T16:52:24,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/70accd13a1e8437f85e272bd7d227a8d, entries=200, sequenceid=396, filesize=14.4 K 2024-11-20T16:52:24,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/b6664a410be248bc956b60c73ac36d49 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b6664a410be248bc956b60c73ac36d49 2024-11-20T16:52:24,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b6664a410be248bc956b60c73ac36d49, entries=150, sequenceid=396, filesize=12.0 K 2024-11-20T16:52:24,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/4af45641b61a43a98d71e25bebc78903 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/4af45641b61a43a98d71e25bebc78903 2024-11-20T16:52:24,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/4af45641b61a43a98d71e25bebc78903, entries=150, sequenceid=396, filesize=12.0 K 2024-11-20T16:52:24,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b683f7af58301e87dbf6f3d58dd983d2 in 130ms, sequenceid=396, compaction requested=true 2024-11-20T16:52:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:24,211 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:52:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:24,211 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:52:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:24,214 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52258 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:52:24,214 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/A is initiating minor compaction (all files) 2024-11-20T16:52:24,214 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/A in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:24,214 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/e3695985298e4e7a8f8c6097650e3d67, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c4bd4b602a4a4d48ae6bf39f289060c2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/4a895c1b259f4c1e8df7f8f4605496be, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/70accd13a1e8437f85e272bd7d227a8d] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=51.0 K 2024-11-20T16:52:24,215 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:52:24,215 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/B is initiating minor compaction (all files) 2024-11-20T16:52:24,215 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/B in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:24,215 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/82326f9270a145fe9bc7ed425180b639, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/ffda9d956a4340b6b4ff3bfcde39660a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/58ea3b9d267b4bef919a46fe630165dc, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b6664a410be248bc956b60c73ac36d49] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=48.7 K 2024-11-20T16:52:24,215 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3695985298e4e7a8f8c6097650e3d67, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732121540032 2024-11-20T16:52:24,215 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 82326f9270a145fe9bc7ed425180b639, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732121540032 2024-11-20T16:52:24,216 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4bd4b602a4a4d48ae6bf39f289060c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732121541150 2024-11-20T16:52:24,216 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting ffda9d956a4340b6b4ff3bfcde39660a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732121541150 2024-11-20T16:52:24,216 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a895c1b259f4c1e8df7f8f4605496be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732121542265 2024-11-20T16:52:24,217 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 58ea3b9d267b4bef919a46fe630165dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732121542265 2024-11-20T16:52:24,217 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70accd13a1e8437f85e272bd7d227a8d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732121542930 2024-11-20T16:52:24,218 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting b6664a410be248bc956b60c73ac36d49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732121542930 2024-11-20T16:52:24,236 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:24,237 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/c2b57ae6b02f48b09280ab8022885d99 is 50, key is test_row_0/A:col10/1732121544078/Put/seqid=0 2024-11-20T16:52:24,240 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#B#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:24,241 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/3065b1e1db9b4556ab0ddc09db9e4ed8 is 50, key is test_row_0/B:col10/1732121544078/Put/seqid=0 2024-11-20T16:52:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:24,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T16:52:24,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:24,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:24,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:24,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:24,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:24,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:24,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/bb902c16302940fca3f1e9e07fca9d6e is 50, key is test_row_0/A:col10/1732121544114/Put/seqid=0 2024-11-20T16:52:24,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741904_1080 (size=13051) 2024-11-20T16:52:24,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121604277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:24,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121604278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:24,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121604279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:24,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:24,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:24,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:24,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741903_1079 (size=13051) 2024-11-20T16:52:24,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741905_1081 (size=17181) 2024-11-20T16:52:24,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/bb902c16302940fca3f1e9e07fca9d6e 2024-11-20T16:52:24,303 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/c2b57ae6b02f48b09280ab8022885d99 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c2b57ae6b02f48b09280ab8022885d99 2024-11-20T16:52:24,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/43278fe87017441aaa46327e821868fb is 50, key is test_row_0/B:col10/1732121544114/Put/seqid=0 2024-11-20T16:52:24,313 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/A of b683f7af58301e87dbf6f3d58dd983d2 into c2b57ae6b02f48b09280ab8022885d99(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:24,313 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:24,313 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/A, priority=12, startTime=1732121544211; duration=0sec 2024-11-20T16:52:24,313 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:24,313 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:A 2024-11-20T16:52:24,313 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:52:24,315 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:52:24,315 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/C is initiating minor compaction (all files) 2024-11-20T16:52:24,315 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/C in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:24,315 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/52b669afb9684fc5b8a12921697642b9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/543eaf97df9c494d99f89cac9118af77, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3f330077b8eb43c680b71e24c0131c21, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/4af45641b61a43a98d71e25bebc78903] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=48.7 K 2024-11-20T16:52:24,316 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52b669afb9684fc5b8a12921697642b9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732121540032 2024-11-20T16:52:24,316 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 543eaf97df9c494d99f89cac9118af77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732121541150 2024-11-20T16:52:24,317 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f330077b8eb43c680b71e24c0131c21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732121542265 2024-11-20T16:52:24,317 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4af45641b61a43a98d71e25bebc78903, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732121542930 2024-11-20T16:52:24,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741906_1082 (size=12301) 2024-11-20T16:52:24,338 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#C#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:24,339 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/a79c2131b1e445b7b112c6bba4565c0e is 50, key is test_row_0/C:col10/1732121544078/Put/seqid=0 2024-11-20T16:52:24,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741907_1083 (size=13051) 2024-11-20T16:52:24,357 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/a79c2131b1e445b7b112c6bba4565c0e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/a79c2131b1e445b7b112c6bba4565c0e 2024-11-20T16:52:24,368 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/C of b683f7af58301e87dbf6f3d58dd983d2 into a79c2131b1e445b7b112c6bba4565c0e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:24,368 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:24,368 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/C, priority=12, startTime=1732121544211; duration=0sec 2024-11-20T16:52:24,369 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:24,369 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:C 2024-11-20T16:52:24,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:24,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121604397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121604396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:24,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121604397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:24,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121604606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:24,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121604606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:24,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121604606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:24,690 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/3065b1e1db9b4556ab0ddc09db9e4ed8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/3065b1e1db9b4556ab0ddc09db9e4ed8 2024-11-20T16:52:24,698 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/B of b683f7af58301e87dbf6f3d58dd983d2 into 3065b1e1db9b4556ab0ddc09db9e4ed8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:24,698 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:24,698 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/B, priority=12, startTime=1732121544211; duration=0sec 2024-11-20T16:52:24,699 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:24,699 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:B 2024-11-20T16:52:24,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/43278fe87017441aaa46327e821868fb 2024-11-20T16:52:24,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/f74d9f8e580d44b498c4ade3c9229257 is 50, key is test_row_0/C:col10/1732121544114/Put/seqid=0 2024-11-20T16:52:24,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741908_1084 (size=12301) 2024-11-20T16:52:24,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121604916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:24,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121604916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:24,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121604927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:25,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/f74d9f8e580d44b498c4ade3c9229257 2024-11-20T16:52:25,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/bb902c16302940fca3f1e9e07fca9d6e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/bb902c16302940fca3f1e9e07fca9d6e 2024-11-20T16:52:25,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/bb902c16302940fca3f1e9e07fca9d6e, entries=250, sequenceid=416, filesize=16.8 K 2024-11-20T16:52:25,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/43278fe87017441aaa46327e821868fb as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/43278fe87017441aaa46327e821868fb 2024-11-20T16:52:25,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/43278fe87017441aaa46327e821868fb, entries=150, sequenceid=416, filesize=12.0 K 2024-11-20T16:52:25,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/f74d9f8e580d44b498c4ade3c9229257 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/f74d9f8e580d44b498c4ade3c9229257 2024-11-20T16:52:25,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/f74d9f8e580d44b498c4ade3c9229257, entries=150, sequenceid=416, filesize=12.0 K 2024-11-20T16:52:25,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b683f7af58301e87dbf6f3d58dd983d2 in 934ms, sequenceid=416, compaction requested=false 2024-11-20T16:52:25,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:25,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T16:52:25,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:25,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:25,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:25,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:25,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:25,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:25,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/effbc61cd76f44f3a01debd74815075b is 50, key is test_row_0/A:col10/1732121544272/Put/seqid=0 2024-11-20T16:52:25,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741909_1085 (size=14741) 2024-11-20T16:52:25,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/effbc61cd76f44f3a01debd74815075b 2024-11-20T16:52:25,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/5aa2aff497a24f4c8f65410e48475384 is 50, key is test_row_0/B:col10/1732121544272/Put/seqid=0 2024-11-20T16:52:25,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741910_1086 (size=12301) 2024-11-20T16:52:25,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/5aa2aff497a24f4c8f65410e48475384 2024-11-20T16:52:25,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121605425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:25,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121605426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:25,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121605428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:25,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/fbbdb9a0fd6242bbb4de0887f07ac562 is 50, key is test_row_0/C:col10/1732121544272/Put/seqid=0 2024-11-20T16:52:25,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741911_1087 (size=12301) 2024-11-20T16:52:25,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121605437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:25,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121605437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:25,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121605535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:25,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121605536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T16:52:25,627 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:25,627 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-20T16:52:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T16:52:25,629 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:25,630 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:25,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T16:52:25,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121605746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:25,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121605746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:25,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T16:52:25,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:25,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:25,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:25,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:25,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:25,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:25,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/fbbdb9a0fd6242bbb4de0887f07ac562 2024-11-20T16:52:25,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/effbc61cd76f44f3a01debd74815075b as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/effbc61cd76f44f3a01debd74815075b 2024-11-20T16:52:25,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/effbc61cd76f44f3a01debd74815075b, entries=200, sequenceid=433, filesize=14.4 K 2024-11-20T16:52:25,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/5aa2aff497a24f4c8f65410e48475384 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/5aa2aff497a24f4c8f65410e48475384 2024-11-20T16:52:25,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/5aa2aff497a24f4c8f65410e48475384, entries=150, sequenceid=433, filesize=12.0 K 2024-11-20T16:52:25,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/fbbdb9a0fd6242bbb4de0887f07ac562 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/fbbdb9a0fd6242bbb4de0887f07ac562 2024-11-20T16:52:25,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/fbbdb9a0fd6242bbb4de0887f07ac562, entries=150, sequenceid=433, filesize=12.0 K 2024-11-20T16:52:25,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b683f7af58301e87dbf6f3d58dd983d2 in 490ms, sequenceid=433, compaction requested=true 2024-11-20T16:52:25,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:25,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:25,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:25,870 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:25,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:25,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:25,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:25,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T16:52:25,870 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:25,872 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44973 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:25,872 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/A is initiating minor compaction (all files) 2024-11-20T16:52:25,872 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/A in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:25,872 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c2b57ae6b02f48b09280ab8022885d99, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/bb902c16302940fca3f1e9e07fca9d6e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/effbc61cd76f44f3a01debd74815075b] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=43.9 K 2024-11-20T16:52:25,872 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:25,873 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/B is initiating minor compaction (all files) 2024-11-20T16:52:25,873 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/B in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:25,873 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/3065b1e1db9b4556ab0ddc09db9e4ed8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/43278fe87017441aaa46327e821868fb, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/5aa2aff497a24f4c8f65410e48475384] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=36.8 K 2024-11-20T16:52:25,873 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2b57ae6b02f48b09280ab8022885d99, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732121542930 2024-11-20T16:52:25,873 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 3065b1e1db9b4556ab0ddc09db9e4ed8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732121542930 2024-11-20T16:52:25,874 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb902c16302940fca3f1e9e07fca9d6e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732121544110 2024-11-20T16:52:25,874 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting effbc61cd76f44f3a01debd74815075b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732121544272 2024-11-20T16:52:25,875 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 43278fe87017441aaa46327e821868fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732121544114 2024-11-20T16:52:25,875 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aa2aff497a24f4c8f65410e48475384, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732121544272 2024-11-20T16:52:25,888 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#A#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:25,889 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/d3144c7965414d75943e1e3f27c95c8e is 50, key is test_row_0/A:col10/1732121544272/Put/seqid=0 2024-11-20T16:52:25,899 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#B#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:25,900 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/04fa8078f0ca4c1dbb67db477a325c2b is 50, key is test_row_0/B:col10/1732121544272/Put/seqid=0 2024-11-20T16:52:25,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741912_1088 (size=13153) 2024-11-20T16:52:25,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741913_1089 (size=13153) 2024-11-20T16:52:25,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T16:52:25,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:25,937 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T16:52:25,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:25,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:25,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:25,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:25,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:25,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:25,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/e29e43e7ddc24d2cbcf90bbfed010ddf is 50, key is test_row_0/A:col10/1732121545423/Put/seqid=0 2024-11-20T16:52:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T16:52:25,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741914_1090 (size=12301) 2024-11-20T16:52:25,956 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/e29e43e7ddc24d2cbcf90bbfed010ddf 2024-11-20T16:52:25,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/a93def6102684a98abebb8202f71d5b1 is 50, key is test_row_0/B:col10/1732121545423/Put/seqid=0 2024-11-20T16:52:25,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741915_1091 (size=12301) 2024-11-20T16:52:25,977 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/a93def6102684a98abebb8202f71d5b1 2024-11-20T16:52:25,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/74202a971f104e268fe94e3fb0622d51 is 50, key is test_row_0/C:col10/1732121545423/Put/seqid=0 2024-11-20T16:52:25,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741916_1092 (size=12301) 2024-11-20T16:52:26,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:26,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:26,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121606077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121606080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121606186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121606196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T16:52:26,308 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/d3144c7965414d75943e1e3f27c95c8e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/d3144c7965414d75943e1e3f27c95c8e 2024-11-20T16:52:26,318 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/04fa8078f0ca4c1dbb67db477a325c2b as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/04fa8078f0ca4c1dbb67db477a325c2b 2024-11-20T16:52:26,320 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/A of b683f7af58301e87dbf6f3d58dd983d2 into d3144c7965414d75943e1e3f27c95c8e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:26,320 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:26,320 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/A, priority=13, startTime=1732121545869; duration=0sec 2024-11-20T16:52:26,323 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:26,323 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:A 2024-11-20T16:52:26,323 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:26,326 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:26,326 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/C is initiating minor compaction (all files) 2024-11-20T16:52:26,326 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/C in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:26,327 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/a79c2131b1e445b7b112c6bba4565c0e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/f74d9f8e580d44b498c4ade3c9229257, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/fbbdb9a0fd6242bbb4de0887f07ac562] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=36.8 K 2024-11-20T16:52:26,328 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting a79c2131b1e445b7b112c6bba4565c0e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732121542930 2024-11-20T16:52:26,329 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting f74d9f8e580d44b498c4ade3c9229257, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732121544114 2024-11-20T16:52:26,329 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbbdb9a0fd6242bbb4de0887f07ac562, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732121544272 2024-11-20T16:52:26,329 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/B of b683f7af58301e87dbf6f3d58dd983d2 into 04fa8078f0ca4c1dbb67db477a325c2b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:26,329 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:26,329 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/B, priority=13, startTime=1732121545870; duration=0sec 2024-11-20T16:52:26,330 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:26,330 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:B 2024-11-20T16:52:26,338 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#C#compaction#80 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:26,340 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/1fe7c60266234037aa19c8f08d4b5e7a is 50, key is test_row_0/C:col10/1732121544272/Put/seqid=0 2024-11-20T16:52:26,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741917_1093 (size=13153) 2024-11-20T16:52:26,398 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/74202a971f104e268fe94e3fb0622d51 2024-11-20T16:52:26,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121606396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/e29e43e7ddc24d2cbcf90bbfed010ddf as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/e29e43e7ddc24d2cbcf90bbfed010ddf 2024-11-20T16:52:26,412 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/e29e43e7ddc24d2cbcf90bbfed010ddf, entries=150, sequenceid=451, filesize=12.0 K 2024-11-20T16:52:26,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/a93def6102684a98abebb8202f71d5b1 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/a93def6102684a98abebb8202f71d5b1 2024-11-20T16:52:26,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,417 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/a93def6102684a98abebb8202f71d5b1, entries=150, sequenceid=451, filesize=12.0 K 2024-11-20T16:52:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121606410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/74202a971f104e268fe94e3fb0622d51 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/74202a971f104e268fe94e3fb0622d51 2024-11-20T16:52:26,426 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/74202a971f104e268fe94e3fb0622d51, entries=150, sequenceid=451, filesize=12.0 K 2024-11-20T16:52:26,427 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b683f7af58301e87dbf6f3d58dd983d2 in 491ms, sequenceid=451, compaction requested=false 2024-11-20T16:52:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:26,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-20T16:52:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-20T16:52:26,431 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-20T16:52:26,431 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 799 msec 2024-11-20T16:52:26,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 805 msec 2024-11-20T16:52:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:26,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T16:52:26,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:26,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:26,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:26,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:26,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:26,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:26,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/2989b1d7bff449edad2b2f7b4411ce1e is 50, key is test_row_0/A:col10/1732121546073/Put/seqid=0 2024-11-20T16:52:26,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741918_1094 (size=14741) 2024-11-20T16:52:26,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/2989b1d7bff449edad2b2f7b4411ce1e 2024-11-20T16:52:26,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/0edeb2936660444b966c7121e0a9ac94 is 50, key is test_row_0/B:col10/1732121546073/Put/seqid=0 2024-11-20T16:52:26,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741919_1095 (size=12301) 2024-11-20T16:52:26,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/0edeb2936660444b966c7121e0a9ac94 2024-11-20T16:52:26,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121606478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121606480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121606482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/07a0292c19734c45851be953c4ae5810 is 50, key is test_row_0/C:col10/1732121546073/Put/seqid=0 2024-11-20T16:52:26,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741920_1096 (size=12301) 2024-11-20T16:52:26,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/07a0292c19734c45851be953c4ae5810 2024-11-20T16:52:26,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/2989b1d7bff449edad2b2f7b4411ce1e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/2989b1d7bff449edad2b2f7b4411ce1e 2024-11-20T16:52:26,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/2989b1d7bff449edad2b2f7b4411ce1e, entries=200, sequenceid=468, filesize=14.4 K 2024-11-20T16:52:26,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/0edeb2936660444b966c7121e0a9ac94 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/0edeb2936660444b966c7121e0a9ac94 2024-11-20T16:52:26,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/0edeb2936660444b966c7121e0a9ac94, entries=150, sequenceid=468, filesize=12.0 K 2024-11-20T16:52:26,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/07a0292c19734c45851be953c4ae5810 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/07a0292c19734c45851be953c4ae5810 2024-11-20T16:52:26,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/07a0292c19734c45851be953c4ae5810, entries=150, sequenceid=468, filesize=12.0 K 2024-11-20T16:52:26,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b683f7af58301e87dbf6f3d58dd983d2 in 104ms, sequenceid=468, compaction requested=true 2024-11-20T16:52:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:A, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:26,542 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:B, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T16:52:26,543 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:26,543 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/A is initiating minor compaction (all files) 2024-11-20T16:52:26,543 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/A in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:26,544 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/d3144c7965414d75943e1e3f27c95c8e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/e29e43e7ddc24d2cbcf90bbfed010ddf, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/2989b1d7bff449edad2b2f7b4411ce1e] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=39.3 K 2024-11-20T16:52:26,545 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting d3144c7965414d75943e1e3f27c95c8e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732121544272 2024-11-20T16:52:26,546 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting e29e43e7ddc24d2cbcf90bbfed010ddf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732121545420 2024-11-20T16:52:26,548 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 2989b1d7bff449edad2b2f7b4411ce1e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1732121546069 2024-11-20T16:52:26,561 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#A#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:26,561 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/30dafc9b3b664390bae78a553e931cb6 is 50, key is test_row_0/A:col10/1732121546073/Put/seqid=0 2024-11-20T16:52:26,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741921_1097 (size=13255) 2024-11-20T16:52:26,580 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/30dafc9b3b664390bae78a553e931cb6 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/30dafc9b3b664390bae78a553e931cb6 2024-11-20T16:52:26,590 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/A of b683f7af58301e87dbf6f3d58dd983d2 into 30dafc9b3b664390bae78a553e931cb6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:26,590 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:26,590 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/A, priority=13, startTime=1732121546542; duration=0sec 2024-11-20T16:52:26,590 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T16:52:26,590 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:A 2024-11-20T16:52:26,591 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-20T16:52:26,592 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T16:52:26,592 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T16:52:26,592 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. because compaction request was cancelled 2024-11-20T16:52:26,592 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:C 2024-11-20T16:52:26,592 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:26,593 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:26,593 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/B is initiating minor compaction (all files) 2024-11-20T16:52:26,593 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/B in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:26,593 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/04fa8078f0ca4c1dbb67db477a325c2b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/a93def6102684a98abebb8202f71d5b1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/0edeb2936660444b966c7121e0a9ac94] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=36.9 K 2024-11-20T16:52:26,594 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 04fa8078f0ca4c1dbb67db477a325c2b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732121544272 2024-11-20T16:52:26,594 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting a93def6102684a98abebb8202f71d5b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732121545420 2024-11-20T16:52:26,594 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 0edeb2936660444b966c7121e0a9ac94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1732121546073 2024-11-20T16:52:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:26,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T16:52:26,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:26,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:26,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:26,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:26,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:26,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:26,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/3006f6f4105943209017b68816984f4c is 50, key is test_row_0/A:col10/1732121546477/Put/seqid=0 2024-11-20T16:52:26,616 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#B#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:26,617 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/d4f3201d375d47008542e7385f8d2ef8 is 50, key is test_row_0/B:col10/1732121546073/Put/seqid=0 2024-11-20T16:52:26,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741922_1098 (size=17181) 2024-11-20T16:52:26,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/3006f6f4105943209017b68816984f4c 2024-11-20T16:52:26,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121606622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121606628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121606630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741923_1099 (size=13255) 2024-11-20T16:52:26,649 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/d4f3201d375d47008542e7385f8d2ef8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/d4f3201d375d47008542e7385f8d2ef8 2024-11-20T16:52:26,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/9df0380087184ee7b49738bca66561a2 is 50, key is test_row_0/B:col10/1732121546477/Put/seqid=0 2024-11-20T16:52:26,660 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/B of b683f7af58301e87dbf6f3d58dd983d2 into d4f3201d375d47008542e7385f8d2ef8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:26,660 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:26,660 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/B, priority=13, startTime=1732121546542; duration=0sec 2024-11-20T16:52:26,660 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:26,660 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:B 2024-11-20T16:52:26,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741924_1100 (size=12301) 2024-11-20T16:52:26,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/9df0380087184ee7b49738bca66561a2 2024-11-20T16:52:26,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/9bf0ca944e3546fbb10c1835530bc934 is 50, key is test_row_0/C:col10/1732121546477/Put/seqid=0 2024-11-20T16:52:26,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741925_1101 (size=12301) 2024-11-20T16:52:26,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/9bf0ca944e3546fbb10c1835530bc934 2024-11-20T16:52:26,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121606706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/3006f6f4105943209017b68816984f4c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/3006f6f4105943209017b68816984f4c 2024-11-20T16:52:26,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/3006f6f4105943209017b68816984f4c, entries=250, sequenceid=488, filesize=16.8 K 2024-11-20T16:52:26,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/9df0380087184ee7b49738bca66561a2 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/9df0380087184ee7b49738bca66561a2 2024-11-20T16:52:26,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121606726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/9df0380087184ee7b49738bca66561a2, entries=150, sequenceid=488, filesize=12.0 K 2024-11-20T16:52:26,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/9bf0ca944e3546fbb10c1835530bc934 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/9bf0ca944e3546fbb10c1835530bc934 2024-11-20T16:52:26,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121606747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121606747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/9bf0ca944e3546fbb10c1835530bc934, entries=150, sequenceid=488, filesize=12.0 K 2024-11-20T16:52:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121606747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b683f7af58301e87dbf6f3d58dd983d2 in 158ms, sequenceid=488, compaction requested=true 2024-11-20T16:52:26,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:26,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:26,754 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T16:52:26,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:26,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:26,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:26,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:26,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T16:52:26,754 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T16:52:26,754 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T16:52:26,754 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. because compaction request was cancelled 2024-11-20T16:52:26,755 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:A 2024-11-20T16:52:26,755 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 3 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:26,756 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:26,756 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/C is initiating minor compaction 2024-11-20T16:52:26,756 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/C in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:26,756 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/74202a971f104e268fe94e3fb0622d51, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/07a0292c19734c45851be953c4ae5810, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/9bf0ca944e3546fbb10c1835530bc934] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=36.0 K 2024-11-20T16:52:26,756 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 74202a971f104e268fe94e3fb0622d51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451 2024-11-20T16:52:26,756 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 07a0292c19734c45851be953c4ae5810, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468 2024-11-20T16:52:26,757 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bf0ca944e3546fbb10c1835530bc934, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488 2024-11-20T16:52:26,762 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/1fe7c60266234037aa19c8f08d4b5e7a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/1fe7c60266234037aa19c8f08d4b5e7a 2024-11-20T16:52:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T16:52:26,769 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:26,770 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-20T16:52:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T16:52:26,773 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:26,775 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:26,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:26,777 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/C of b683f7af58301e87dbf6f3d58dd983d2 into 1fe7c60266234037aa19c8f08d4b5e7a(size=12.8 K), total size for store is 48.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:26,777 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:26,778 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/C, priority=13, startTime=1732121545870; duration=0sec 2024-11-20T16:52:26,778 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:26,778 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:C 2024-11-20T16:52:26,778 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T16:52:26,780 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T16:52:26,780 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T16:52:26,780 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. because compaction request was cancelled 2024-11-20T16:52:26,780 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:B 2024-11-20T16:52:26,782 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#C#compaction#89 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:26,782 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/95abed4a85044fd6adf3201bad308e8c is 50, key is test_row_0/C:col10/1732121546477/Put/seqid=0 2024-11-20T16:52:26,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741926_1102 (size=12404) 2024-11-20T16:52:26,826 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/95abed4a85044fd6adf3201bad308e8c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/95abed4a85044fd6adf3201bad308e8c 2024-11-20T16:52:26,835 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 file(s) in b683f7af58301e87dbf6f3d58dd983d2/C of b683f7af58301e87dbf6f3d58dd983d2 into 95abed4a85044fd6adf3201bad308e8c(size=12.1 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:26,835 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:26,835 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/C, priority=10, startTime=1732121546754; duration=0sec 2024-11-20T16:52:26,836 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:26,836 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:C 2024-11-20T16:52:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T16:52:26,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T16:52:26,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:26,929 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T16:52:26,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:26,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:26,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:26,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:26,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:26,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:26,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/891ecf8fac3741cca7ebe1ddfeb29ab3 is 50, key is test_row_0/A:col10/1732121546627/Put/seqid=0 2024-11-20T16:52:26,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741927_1103 (size=12301) 2024-11-20T16:52:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:26,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:26,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121606985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121606988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:26,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121606992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:26,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T16:52:27,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121607096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:27,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121607097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:27,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:27,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121607106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:27,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121607216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:27,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:27,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121607236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:27,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121607306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:27,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121607306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:27,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:27,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121607315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:27,357 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/891ecf8fac3741cca7ebe1ddfeb29ab3 2024-11-20T16:52:27,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/5c8afd7cf21f46f9b6ae60e3f9819c82 is 50, key is test_row_0/B:col10/1732121546627/Put/seqid=0 2024-11-20T16:52:27,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741928_1104 (size=12301) 2024-11-20T16:52:27,393 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/5c8afd7cf21f46f9b6ae60e3f9819c82 2024-11-20T16:52:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T16:52:27,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/34bf0deebf194ea2a625b123ffe04ca9 is 50, key is test_row_0/C:col10/1732121546627/Put/seqid=0 2024-11-20T16:52:27,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741929_1105 (size=12301) 2024-11-20T16:52:27,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121607616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121607616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:27,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121607626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:27,829 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/34bf0deebf194ea2a625b123ffe04ca9 2024-11-20T16:52:27,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/891ecf8fac3741cca7ebe1ddfeb29ab3 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/891ecf8fac3741cca7ebe1ddfeb29ab3 2024-11-20T16:52:27,842 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/891ecf8fac3741cca7ebe1ddfeb29ab3, entries=150, sequenceid=504, filesize=12.0 K 2024-11-20T16:52:27,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/5c8afd7cf21f46f9b6ae60e3f9819c82 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/5c8afd7cf21f46f9b6ae60e3f9819c82 2024-11-20T16:52:27,851 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/5c8afd7cf21f46f9b6ae60e3f9819c82, entries=150, sequenceid=504, filesize=12.0 K 2024-11-20T16:52:27,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/34bf0deebf194ea2a625b123ffe04ca9 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/34bf0deebf194ea2a625b123ffe04ca9 2024-11-20T16:52:27,858 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/34bf0deebf194ea2a625b123ffe04ca9, entries=150, sequenceid=504, filesize=12.0 K 2024-11-20T16:52:27,859 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b683f7af58301e87dbf6f3d58dd983d2 in 930ms, sequenceid=504, compaction requested=true 2024-11-20T16:52:27,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:27,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:27,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-20T16:52:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-20T16:52:27,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-20T16:52:27,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0860 sec 2024-11-20T16:52:27,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.0940 sec 2024-11-20T16:52:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T16:52:27,917 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:27,918 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-20T16:52:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T16:52:27,920 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:27,921 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:27,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T16:52:28,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T16:52:28,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:28,075 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T16:52:28,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:28,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:28,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:28,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:28,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:28,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:28,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/16c1690132ed413a929680660a8feab9 is 50, key is test_row_0/A:col10/1732121546990/Put/seqid=0 2024-11-20T16:52:28,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741930_1106 (size=12301) 2024-11-20T16:52:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:28,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:28,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121608149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:28,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:28,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121608149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:28,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:28,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121608150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:28,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:28,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121608226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:28,232 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in b683f7af58301e87dbf6f3d58dd983d2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:52:28,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T16:52:28,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121608246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:28,249 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_0' in b683f7af58301e87dbf6f3d58dd983d2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:52:28,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:28,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:28,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121608266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:28,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121608266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:28,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121608266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:28,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:28,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121608475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:28,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:28,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121608476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:28,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:28,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:28,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121608476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:28,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:28,487 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/16c1690132ed413a929680660a8feab9 2024-11-20T16:52:28,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/51119fc0e4b0483f9bb0a9f405c27146 is 50, key is test_row_0/B:col10/1732121546990/Put/seqid=0 2024-11-20T16:52:28,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741931_1107 (size=12301) 2024-11-20T16:52:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T16:52:28,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121608801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:28,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121608801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:28,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121608801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:28,906 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/51119fc0e4b0483f9bb0a9f405c27146 2024-11-20T16:52:28,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/804ad9f3e20540e89c6f30d3e562e152 is 50, key is test_row_0/C:col10/1732121546990/Put/seqid=0 2024-11-20T16:52:28,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741932_1108 (size=12301) 2024-11-20T16:52:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T16:52:29,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121609307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:29,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121609307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:29,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121609316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:29,321 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/804ad9f3e20540e89c6f30d3e562e152 2024-11-20T16:52:29,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/16c1690132ed413a929680660a8feab9 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/16c1690132ed413a929680660a8feab9 2024-11-20T16:52:29,335 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/16c1690132ed413a929680660a8feab9, entries=150, sequenceid=523, filesize=12.0 K 2024-11-20T16:52:29,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/51119fc0e4b0483f9bb0a9f405c27146 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/51119fc0e4b0483f9bb0a9f405c27146 2024-11-20T16:52:29,343 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/51119fc0e4b0483f9bb0a9f405c27146, entries=150, sequenceid=523, filesize=12.0 K 2024-11-20T16:52:29,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/804ad9f3e20540e89c6f30d3e562e152 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/804ad9f3e20540e89c6f30d3e562e152 2024-11-20T16:52:29,361 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/804ad9f3e20540e89c6f30d3e562e152, entries=150, sequenceid=523, filesize=12.0 K 2024-11-20T16:52:29,362 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b683f7af58301e87dbf6f3d58dd983d2 in 1287ms, sequenceid=523, compaction requested=true 2024-11-20T16:52:29,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:29,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:29,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-20T16:52:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-20T16:52:29,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-20T16:52:29,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4440 sec 2024-11-20T16:52:29,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.4500 sec 2024-11-20T16:52:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T16:52:30,066 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:30,067 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-20T16:52:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T16:52:30,068 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:30,069 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:30,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:30,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T16:52:30,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T16:52:30,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:30,222 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T16:52:30,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:30,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:30,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:30,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:30,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:30,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:30,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/738f2ed0f32f4ec5b9536b73277003dc is 50, key is test_row_0/A:col10/1732121548148/Put/seqid=0 2024-11-20T16:52:30,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741933_1109 (size=12301) 2024-11-20T16:52:30,241 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/738f2ed0f32f4ec5b9536b73277003dc 2024-11-20T16:52:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:30,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:30,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/e12b91e2ee3b41f5b6d959e687d8be6e is 50, key is test_row_0/B:col10/1732121548148/Put/seqid=0 2024-11-20T16:52:30,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741934_1110 (size=12301) 2024-11-20T16:52:30,267 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/e12b91e2ee3b41f5b6d959e687d8be6e 2024-11-20T16:52:30,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/b3b2dd49595a4e9b8871b5ffd60e2875 is 50, key is test_row_0/C:col10/1732121548148/Put/seqid=0 2024-11-20T16:52:30,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741935_1111 (size=12301) 2024-11-20T16:52:30,285 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/b3b2dd49595a4e9b8871b5ffd60e2875 2024-11-20T16:52:30,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/738f2ed0f32f4ec5b9536b73277003dc as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/738f2ed0f32f4ec5b9536b73277003dc 2024-11-20T16:52:30,297 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/738f2ed0f32f4ec5b9536b73277003dc, entries=150, sequenceid=536, filesize=12.0 K 2024-11-20T16:52:30,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/e12b91e2ee3b41f5b6d959e687d8be6e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/e12b91e2ee3b41f5b6d959e687d8be6e 2024-11-20T16:52:30,304 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/e12b91e2ee3b41f5b6d959e687d8be6e, entries=150, sequenceid=536, filesize=12.0 K 2024-11-20T16:52:30,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/b3b2dd49595a4e9b8871b5ffd60e2875 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/b3b2dd49595a4e9b8871b5ffd60e2875 2024-11-20T16:52:30,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121610304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,314 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/b3b2dd49595a4e9b8871b5ffd60e2875, entries=150, sequenceid=536, filesize=12.0 K 2024-11-20T16:52:30,316 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b683f7af58301e87dbf6f3d58dd983d2 in 94ms, sequenceid=536, compaction requested=true 2024-11-20T16:52:30,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:30,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:30,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-20T16:52:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-20T16:52:30,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T16:52:30,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:30,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:30,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:30,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:30,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:30,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:30,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-20T16:52:30,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 250 msec 2024-11-20T16:52:30,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 258 msec 2024-11-20T16:52:30,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/b5488cbcdd3742409f8bc65ddcb586b9 is 50, key is test_row_0/A:col10/1732121550306/Put/seqid=0 2024-11-20T16:52:30,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741936_1112 (size=14741) 2024-11-20T16:52:30,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121610339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121610340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121610344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121610346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T16:52:30,386 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:30,386 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:30,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-20T16:52:30,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T16:52:30,389 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:30,390 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:30,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:30,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121610416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121610457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121610457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121610457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121610457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T16:52:30,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T16:52:30,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:30,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:30,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:30,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:30,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:30,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:30,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121610626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121610666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121610666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121610666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121610666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T16:52:30,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:30,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:30,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:30,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:30,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T16:52:30,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/b5488cbcdd3742409f8bc65ddcb586b9 2024-11-20T16:52:30,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/d3c5e0fc47fd4bffa23847fa4b580d28 is 50, key is test_row_0/B:col10/1732121550306/Put/seqid=0 2024-11-20T16:52:30,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741937_1113 (size=12301) 2024-11-20T16:52:30,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T16:52:30,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:30,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:30,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:30,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:30,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:30,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:30,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121610935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121610976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121610976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121610986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:30,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121610986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:31,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T16:52:31,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:31,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:31,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:31,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:31,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T16:52:31,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T16:52:31,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:31,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:31,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:31,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:31,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:31,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/d3c5e0fc47fd4bffa23847fa4b580d28 2024-11-20T16:52:31,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/861fc779c0cc42debd239f91df51b8a4 is 50, key is test_row_0/C:col10/1732121550306/Put/seqid=0 2024-11-20T16:52:31,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741938_1114 (size=12301) 2024-11-20T16:52:31,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T16:52:31,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:31,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:31,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:31,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:31,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:31,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121611446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:31,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:31,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:31,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T16:52:31,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:31,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:31,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:31,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:31,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:31,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121611486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:31,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121611486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:31,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121611496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:31,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121611496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T16:52:31,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/861fc779c0cc42debd239f91df51b8a4 2024-11-20T16:52:31,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/b5488cbcdd3742409f8bc65ddcb586b9 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/b5488cbcdd3742409f8bc65ddcb586b9 2024-11-20T16:52:31,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/b5488cbcdd3742409f8bc65ddcb586b9, entries=200, sequenceid=556, filesize=14.4 K 2024-11-20T16:52:31,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/d3c5e0fc47fd4bffa23847fa4b580d28 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/d3c5e0fc47fd4bffa23847fa4b580d28 2024-11-20T16:52:31,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/d3c5e0fc47fd4bffa23847fa4b580d28, entries=150, sequenceid=556, filesize=12.0 K 2024-11-20T16:52:31,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T16:52:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:31,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/861fc779c0cc42debd239f91df51b8a4 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/861fc779c0cc42debd239f91df51b8a4 2024-11-20T16:52:31,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:31,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:31,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/861fc779c0cc42debd239f91df51b8a4, entries=150, sequenceid=556, filesize=12.0 K 2024-11-20T16:52:31,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b683f7af58301e87dbf6f3d58dd983d2 in 1307ms, sequenceid=556, compaction requested=true 2024-11-20T16:52:31,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:31,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:31,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:31,627 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-20T16:52:31,627 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-20T16:52:31,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:31,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:31,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:31,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:31,629 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74760 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-20T16:52:31,629 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/B is initiating minor compaction (all files) 2024-11-20T16:52:31,629 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/B in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:31,629 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/d4f3201d375d47008542e7385f8d2ef8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/9df0380087184ee7b49738bca66561a2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/5c8afd7cf21f46f9b6ae60e3f9819c82, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/51119fc0e4b0483f9bb0a9f405c27146, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/e12b91e2ee3b41f5b6d959e687d8be6e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/d3c5e0fc47fd4bffa23847fa4b580d28] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=73.0 K 2024-11-20T16:52:31,630 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 82080 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-20T16:52:31,630 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/A is initiating minor compaction (all files) 2024-11-20T16:52:31,630 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/A in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:31,630 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/30dafc9b3b664390bae78a553e931cb6, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/3006f6f4105943209017b68816984f4c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/891ecf8fac3741cca7ebe1ddfeb29ab3, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/16c1690132ed413a929680660a8feab9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/738f2ed0f32f4ec5b9536b73277003dc, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/b5488cbcdd3742409f8bc65ddcb586b9] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=80.2 K 2024-11-20T16:52:31,630 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting d4f3201d375d47008542e7385f8d2ef8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1732121546073 2024-11-20T16:52:31,630 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30dafc9b3b664390bae78a553e931cb6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1732121546073 2024-11-20T16:52:31,631 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 9df0380087184ee7b49738bca66561a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1732121546460 2024-11-20T16:52:31,631 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3006f6f4105943209017b68816984f4c, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1732121546460 2024-11-20T16:52:31,631 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c8afd7cf21f46f9b6ae60e3f9819c82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732121546613 2024-11-20T16:52:31,631 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 891ecf8fac3741cca7ebe1ddfeb29ab3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732121546613 2024-11-20T16:52:31,632 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 51119fc0e4b0483f9bb0a9f405c27146, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732121546973 2024-11-20T16:52:31,632 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16c1690132ed413a929680660a8feab9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732121546973 2024-11-20T16:52:31,632 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting e12b91e2ee3b41f5b6d959e687d8be6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=536, earliestPutTs=1732121548147 2024-11-20T16:52:31,632 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 738f2ed0f32f4ec5b9536b73277003dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=536, earliestPutTs=1732121548147 2024-11-20T16:52:31,632 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting d3c5e0fc47fd4bffa23847fa4b580d28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1732121550295 2024-11-20T16:52:31,632 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5488cbcdd3742409f8bc65ddcb586b9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1732121550295 2024-11-20T16:52:31,645 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#B#compaction#102 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:31,647 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/de9177e254f647d3a743058da6e521a7 is 50, key is test_row_0/B:col10/1732121550306/Put/seqid=0 2024-11-20T16:52:31,653 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#A#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:31,653 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/37ea35801f504a59aa17365bca96afb9 is 50, key is test_row_0/A:col10/1732121550306/Put/seqid=0 2024-11-20T16:52:31,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741939_1115 (size=13459) 2024-11-20T16:52:31,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741940_1116 (size=13459) 2024-11-20T16:52:31,707 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/de9177e254f647d3a743058da6e521a7 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/de9177e254f647d3a743058da6e521a7 2024-11-20T16:52:31,710 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/37ea35801f504a59aa17365bca96afb9 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/37ea35801f504a59aa17365bca96afb9 2024-11-20T16:52:31,719 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/B of b683f7af58301e87dbf6f3d58dd983d2 into de9177e254f647d3a743058da6e521a7(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:31,719 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:31,719 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/B, priority=10, startTime=1732121551627; duration=0sec 2024-11-20T16:52:31,719 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:31,719 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:B 2024-11-20T16:52:31,720 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-20T16:52:31,720 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/A of b683f7af58301e87dbf6f3d58dd983d2 into 37ea35801f504a59aa17365bca96afb9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:31,721 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:31,721 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/A, priority=10, startTime=1732121551627; duration=0sec 2024-11-20T16:52:31,721 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:31,721 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:A 2024-11-20T16:52:31,722 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74761 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-20T16:52:31,723 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/C is initiating minor compaction (all files) 2024-11-20T16:52:31,723 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/C in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:31,723 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/1fe7c60266234037aa19c8f08d4b5e7a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/95abed4a85044fd6adf3201bad308e8c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/34bf0deebf194ea2a625b123ffe04ca9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/804ad9f3e20540e89c6f30d3e562e152, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/b3b2dd49595a4e9b8871b5ffd60e2875, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/861fc779c0cc42debd239f91df51b8a4] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=73.0 K 2024-11-20T16:52:31,723 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fe7c60266234037aa19c8f08d4b5e7a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732121544272 2024-11-20T16:52:31,724 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 95abed4a85044fd6adf3201bad308e8c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1732121546460 2024-11-20T16:52:31,724 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 34bf0deebf194ea2a625b123ffe04ca9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732121546613 2024-11-20T16:52:31,725 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 804ad9f3e20540e89c6f30d3e562e152, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732121546973 2024-11-20T16:52:31,725 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting b3b2dd49595a4e9b8871b5ffd60e2875, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=536, earliestPutTs=1732121548147 2024-11-20T16:52:31,726 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 861fc779c0cc42debd239f91df51b8a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1732121550295 2024-11-20T16:52:31,755 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#C#compaction#104 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:31,756 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/b32ab5d3e21c4467852581930b671283 is 50, key is test_row_0/C:col10/1732121550306/Put/seqid=0 2024-11-20T16:52:31,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741941_1117 (size=13459) 2024-11-20T16:52:31,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T16:52:31,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:31,772 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T16:52:31,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:31,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:31,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:31,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:31,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:31,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:31,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/13dda130581f4092af37b14e5fee21ae is 50, key is test_row_0/A:col10/1732121550343/Put/seqid=0 2024-11-20T16:52:31,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741942_1118 (size=12301) 2024-11-20T16:52:31,785 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/13dda130581f4092af37b14e5fee21ae 2024-11-20T16:52:31,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/b656a404646644d6af43286ecdc3c17b is 50, key is test_row_0/B:col10/1732121550343/Put/seqid=0 2024-11-20T16:52:31,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741943_1119 (size=12301) 2024-11-20T16:52:31,800 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/b656a404646644d6af43286ecdc3c17b 2024-11-20T16:52:31,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/fb58f2bedf7d47c2b6da722cd9ef4e31 is 50, key is test_row_0/C:col10/1732121550343/Put/seqid=0 2024-11-20T16:52:31,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741944_1120 (size=12301) 2024-11-20T16:52:31,816 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/fb58f2bedf7d47c2b6da722cd9ef4e31 2024-11-20T16:52:31,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/13dda130581f4092af37b14e5fee21ae as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/13dda130581f4092af37b14e5fee21ae 2024-11-20T16:52:31,829 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/13dda130581f4092af37b14e5fee21ae, entries=150, sequenceid=570, filesize=12.0 K 2024-11-20T16:52:31,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/b656a404646644d6af43286ecdc3c17b as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b656a404646644d6af43286ecdc3c17b 2024-11-20T16:52:31,842 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b656a404646644d6af43286ecdc3c17b, entries=150, sequenceid=570, filesize=12.0 K 2024-11-20T16:52:31,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/fb58f2bedf7d47c2b6da722cd9ef4e31 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/fb58f2bedf7d47c2b6da722cd9ef4e31 2024-11-20T16:52:31,853 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/fb58f2bedf7d47c2b6da722cd9ef4e31, entries=150, sequenceid=570, filesize=12.0 K 2024-11-20T16:52:31,854 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for b683f7af58301e87dbf6f3d58dd983d2 in 82ms, sequenceid=570, compaction requested=false 2024-11-20T16:52:31,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:31,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:31,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-20T16:52:31,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-20T16:52:31,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-20T16:52:31,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4650 sec 2024-11-20T16:52:31,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.4710 sec 2024-11-20T16:52:32,184 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/b32ab5d3e21c4467852581930b671283 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/b32ab5d3e21c4467852581930b671283 2024-11-20T16:52:32,194 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/C of b683f7af58301e87dbf6f3d58dd983d2 into b32ab5d3e21c4467852581930b671283(size=13.1 K), total size for store is 25.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:32,195 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:32,195 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/C, priority=10, startTime=1732121551627; duration=0sec 2024-11-20T16:52:32,195 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:32,195 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:C 2024-11-20T16:52:32,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:32,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T16:52:32,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:32,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:32,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:32,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:32,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:32,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:32,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/affe1640b02042d8b4004dc4bcef7d5a is 50, key is test_row_0/A:col10/1732121552474/Put/seqid=0 2024-11-20T16:52:32,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741945_1121 (size=17181) 2024-11-20T16:52:32,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732121552507, max timestamp=1732121552521Num uniques 859; 2024-11-20T16:52:32,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T16:52:32,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732121552507, max timestamp=1732121552521Num uniques 859; 2024-11-20T16:52:32,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T16:52:32,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732121552507, max timestamp=1732121552521Num uniques 859; 2024-11-20T16:52:32,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T16:52:32,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T16:52:32,537 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:32,537 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-20T16:52:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T16:52:32,540 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:32,541 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:32,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:32,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:32,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121612545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:32,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121612548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:32,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121612549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:32,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:32,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121612551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:32,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121612552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T16:52:32,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121612655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121612655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:32,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121612656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:32,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:32,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121612666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:32,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121612666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:32,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-20T16:52:32,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:32,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:32,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:32,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:32,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:32,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-20T16:52:32,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:32,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:32,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:32,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:32,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:32,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T16:52:32,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:32,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121612866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:32,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:32,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121612866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:32,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121612866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:32,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:32,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121612876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121612876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:32,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:32,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:32,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:32,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=581 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/affe1640b02042d8b4004dc4bcef7d5a 2024-11-20T16:52:32,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/aa9142eb313f425a8cb94ad5e25ec7b9 is 50, key is test_row_0/B:col10/1732121552474/Put/seqid=0 2024-11-20T16:52:32,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741946_1122 (size=12301) 2024-11-20T16:52:33,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-20T16:52:33,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:33,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:33,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:33,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:33,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:33,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-20T16:52:33,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:33,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:33,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:33,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:33,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:33,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:33,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T16:52:33,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121613176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:33,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:33,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121613176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:33,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121613176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:33,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:33,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121613186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:33,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:33,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:33,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121613195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:33,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:33,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-20T16:52:33,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:33,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:33,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:33,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:33,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:33,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:33,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=581 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/aa9142eb313f425a8cb94ad5e25ec7b9 2024-11-20T16:52:33,313 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-20T16:52:33,313 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:52:33,316 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-20T16:52:33,316 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:52:33,319 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:33,319 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:33,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/3db645fbeada44f386f73ff915913087 is 50, key is test_row_0/C:col10/1732121552474/Put/seqid=0 2024-11-20T16:52:33,320 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:33,320 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:33,320 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:52:33,320 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:52:33,321 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-20T16:52:33,321 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:52:33,321 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:33,321 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:33,321 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:52:33,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741947_1123 (size=12301) 2024-11-20T16:52:33,325 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-20T16:52:33,325 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:52:33,325 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:33,325 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:33,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:52:33,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-20T16:52:33,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:33,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:33,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:33,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:33,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:33,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-20T16:52:33,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:33,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:33,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:33,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:33,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T16:52:33,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:33,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59472 deadline: 1732121613686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59458 deadline: 1732121613686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:33,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:33,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:33,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1732121613696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:33,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:33,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:33,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59476 deadline: 1732121613706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59510 deadline: 1732121613706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:33,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:33,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:33,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:52:33,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=581 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/3db645fbeada44f386f73ff915913087 2024-11-20T16:52:33,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/affe1640b02042d8b4004dc4bcef7d5a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/affe1640b02042d8b4004dc4bcef7d5a 2024-11-20T16:52:33,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/affe1640b02042d8b4004dc4bcef7d5a, entries=250, sequenceid=581, filesize=16.8 K 2024-11-20T16:52:33,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/aa9142eb313f425a8cb94ad5e25ec7b9 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/aa9142eb313f425a8cb94ad5e25ec7b9 2024-11-20T16:52:33,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/aa9142eb313f425a8cb94ad5e25ec7b9, entries=150, sequenceid=581, filesize=12.0 K 2024-11-20T16:52:33,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/3db645fbeada44f386f73ff915913087 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3db645fbeada44f386f73ff915913087 2024-11-20T16:52:33,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3db645fbeada44f386f73ff915913087, entries=150, sequenceid=581, filesize=12.0 K 2024-11-20T16:52:33,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for b683f7af58301e87dbf6f3d58dd983d2 in 1271ms, sequenceid=581, compaction requested=true 2024-11-20T16:52:33,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:33,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:33,746 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:33,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:33,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:33,746 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:33,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:33,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b683f7af58301e87dbf6f3d58dd983d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:33,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:33,747 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:33,747 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:33,747 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/B is initiating minor compaction (all files) 2024-11-20T16:52:33,747 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/A is initiating minor compaction (all files) 2024-11-20T16:52:33,747 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/B in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:33,747 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/A in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:33,748 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/de9177e254f647d3a743058da6e521a7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b656a404646644d6af43286ecdc3c17b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/aa9142eb313f425a8cb94ad5e25ec7b9] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=37.2 K 2024-11-20T16:52:33,748 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/37ea35801f504a59aa17365bca96afb9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/13dda130581f4092af37b14e5fee21ae, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/affe1640b02042d8b4004dc4bcef7d5a] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=41.9 K 2024-11-20T16:52:33,748 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 37ea35801f504a59aa17365bca96afb9, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1732121550295 2024-11-20T16:52:33,748 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting de9177e254f647d3a743058da6e521a7, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1732121550295 2024-11-20T16:52:33,748 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 13dda130581f4092af37b14e5fee21ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1732121550326 2024-11-20T16:52:33,748 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting b656a404646644d6af43286ecdc3c17b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1732121550326 2024-11-20T16:52:33,749 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting affe1640b02042d8b4004dc4bcef7d5a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=581, earliestPutTs=1732121552459 2024-11-20T16:52:33,749 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa9142eb313f425a8cb94ad5e25ec7b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=581, earliestPutTs=1732121552469 2024-11-20T16:52:33,758 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#B#compaction#111 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:33,759 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/a56b65c468d34acdb7e9a709a52bf260 is 50, key is test_row_0/B:col10/1732121552474/Put/seqid=0 2024-11-20T16:52:33,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741948_1124 (size=13561) 2024-11-20T16:52:33,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-20T16:52:33,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:33,768 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-20T16:52:33,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:33,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:33,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:33,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:33,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:33,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:33,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/f30e55d05d804586b57db10d2f072dab is 50, key is test_row_0/A:col10/1732121552551/Put/seqid=0 2024-11-20T16:52:33,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741949_1125 (size=12301) 2024-11-20T16:52:33,784 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#A#compaction#112 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:33,785 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/5d2fff634d4645ea8439836429583669 is 50, key is test_row_0/A:col10/1732121552474/Put/seqid=0 2024-11-20T16:52:33,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741950_1126 (size=13561) 2024-11-20T16:52:34,174 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/a56b65c468d34acdb7e9a709a52bf260 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/a56b65c468d34acdb7e9a709a52bf260 2024-11-20T16:52:34,181 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/B of b683f7af58301e87dbf6f3d58dd983d2 into a56b65c468d34acdb7e9a709a52bf260(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:34,181 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:34,181 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/B, priority=13, startTime=1732121553746; duration=0sec 2024-11-20T16:52:34,182 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:34,182 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:B 2024-11-20T16:52:34,182 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:34,183 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:34,183 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): b683f7af58301e87dbf6f3d58dd983d2/C is initiating minor compaction (all files) 2024-11-20T16:52:34,183 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b683f7af58301e87dbf6f3d58dd983d2/C in TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:34,183 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/b32ab5d3e21c4467852581930b671283, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/fb58f2bedf7d47c2b6da722cd9ef4e31, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3db645fbeada44f386f73ff915913087] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp, totalSize=37.2 K 2024-11-20T16:52:34,184 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting b32ab5d3e21c4467852581930b671283, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1732121550295 2024-11-20T16:52:34,184 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb58f2bedf7d47c2b6da722cd9ef4e31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1732121550326 2024-11-20T16:52:34,185 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/f30e55d05d804586b57db10d2f072dab 2024-11-20T16:52:34,185 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3db645fbeada44f386f73ff915913087, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=581, earliestPutTs=1732121552469 2024-11-20T16:52:34,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/3e4b5cf68f634baaba9d4bf8076a881a is 50, key is test_row_0/B:col10/1732121552551/Put/seqid=0 2024-11-20T16:52:34,200 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/5d2fff634d4645ea8439836429583669 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/5d2fff634d4645ea8439836429583669 2024-11-20T16:52:34,207 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/A of b683f7af58301e87dbf6f3d58dd983d2 into 5d2fff634d4645ea8439836429583669(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:34,207 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:34,207 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/A, priority=13, startTime=1732121553746; duration=0sec 2024-11-20T16:52:34,207 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:34,207 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:A 2024-11-20T16:52:34,209 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b683f7af58301e87dbf6f3d58dd983d2#C#compaction#115 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:34,209 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/3c6848464d594c0f88775f1da29f9b8e is 50, key is test_row_0/C:col10/1732121552474/Put/seqid=0 2024-11-20T16:52:34,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741951_1127 (size=12301) 2024-11-20T16:52:34,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741952_1128 (size=13561) 2024-11-20T16:52:34,613 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/3e4b5cf68f634baaba9d4bf8076a881a 2024-11-20T16:52:34,621 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/3c6848464d594c0f88775f1da29f9b8e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3c6848464d594c0f88775f1da29f9b8e 2024-11-20T16:52:34,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/84ccd4d5e86c4acc950e6e8f567458ed is 50, key is test_row_0/C:col10/1732121552551/Put/seqid=0 2024-11-20T16:52:34,627 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b683f7af58301e87dbf6f3d58dd983d2/C of b683f7af58301e87dbf6f3d58dd983d2 into 3c6848464d594c0f88775f1da29f9b8e(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:34,627 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:34,627 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., storeName=b683f7af58301e87dbf6f3d58dd983d2/C, priority=13, startTime=1732121553746; duration=0sec 2024-11-20T16:52:34,627 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:34,627 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b683f7af58301e87dbf6f3d58dd983d2:C 2024-11-20T16:52:34,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741953_1129 (size=12301) 2024-11-20T16:52:34,673 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T16:52:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T16:52:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:34,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. as already flushing 2024-11-20T16:52:34,698 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-20T16:52:34,698 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:52:34,698 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:34,698 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:34,698 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:52:34,708 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-20T16:52:34,708 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-20T16:52:34,708 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:52:34,708 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:52:34,708 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:34,708 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:34,708 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:34,708 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:34,708 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:52:34,708 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:52:34,717 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-20T16:52:34,717 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-20T16:52:34,717 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:52:34,717 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:34,717 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:52:34,718 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:34,718 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:34,718 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:34,718 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:52:34,718 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:52:35,036 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/84ccd4d5e86c4acc950e6e8f567458ed 2024-11-20T16:52:35,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/f30e55d05d804586b57db10d2f072dab as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/f30e55d05d804586b57db10d2f072dab 2024-11-20T16:52:35,046 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/f30e55d05d804586b57db10d2f072dab, entries=150, sequenceid=616, filesize=12.0 K 2024-11-20T16:52:35,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/3e4b5cf68f634baaba9d4bf8076a881a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/3e4b5cf68f634baaba9d4bf8076a881a 2024-11-20T16:52:35,051 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/3e4b5cf68f634baaba9d4bf8076a881a, entries=150, sequenceid=616, filesize=12.0 K 2024-11-20T16:52:35,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/84ccd4d5e86c4acc950e6e8f567458ed as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/84ccd4d5e86c4acc950e6e8f567458ed 2024-11-20T16:52:35,056 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/84ccd4d5e86c4acc950e6e8f567458ed, entries=150, sequenceid=616, filesize=12.0 K 2024-11-20T16:52:35,057 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=33.54 KB/34350 for b683f7af58301e87dbf6f3d58dd983d2 in 1289ms, sequenceid=616, compaction requested=false 2024-11-20T16:52:35,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for b683f7af58301e87dbf6f3d58dd983d2: 2024-11-20T16:52:35,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:35,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-20T16:52:35,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-20T16:52:35,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-20T16:52:35,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5180 sec 2024-11-20T16:52:35,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 2.5240 sec 2024-11-20T16:52:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T16:52:36,707 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:36,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T16:52:36,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-11-20T16:52:36,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-20T16:52:36,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-20T16:52:36,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-20T16:52:36,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-11-20T16:52:36,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T16:52:36,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2632 2024-11-20T16:52:36,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2535 2024-11-20T16:52:36,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T16:52:36,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1154 2024-11-20T16:52:36,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3459 rows 2024-11-20T16:52:36,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1166 2024-11-20T16:52:36,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3498 rows 2024-11-20T16:52:36,708 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T16:52:36,708 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T16:52:36,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:36,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:36,708 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:52:36,712 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T16:52:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T16:52:36,718 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121556718"}]},"ts":"1732121556718"} 2024-11-20T16:52:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T16:52:36,720 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T16:52:36,721 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T16:52:36,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T16:52:36,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b683f7af58301e87dbf6f3d58dd983d2, UNASSIGN}] 2024-11-20T16:52:36,727 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b683f7af58301e87dbf6f3d58dd983d2, UNASSIGN 2024-11-20T16:52:36,729 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=b683f7af58301e87dbf6f3d58dd983d2, regionState=CLOSING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:36,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b683f7af58301e87dbf6f3d58dd983d2, UNASSIGN because future has completed 2024-11-20T16:52:36,731 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T16:52:36,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:52:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T16:52:36,888 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:36,888 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T16:52:36,889 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing b683f7af58301e87dbf6f3d58dd983d2, disabling compactions & flushes 2024-11-20T16:52:36,889 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:36,889 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:36,889 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. after waiting 0 ms 2024-11-20T16:52:36,889 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:36,889 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing b683f7af58301e87dbf6f3d58dd983d2 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-20T16:52:36,889 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=A 2024-11-20T16:52:36,889 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:36,889 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=B 2024-11-20T16:52:36,889 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:36,890 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b683f7af58301e87dbf6f3d58dd983d2, store=C 2024-11-20T16:52:36,890 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:36,894 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/d0beb500515449e7b38344adb3f5aa8e is 50, key is test_row_0/A:col10/1732121554707/Put/seqid=0 2024-11-20T16:52:36,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741954_1130 (size=12301) 2024-11-20T16:52:37,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T16:52:37,298 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/d0beb500515449e7b38344adb3f5aa8e 2024-11-20T16:52:37,305 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/469b079f936d45528e49c5ece7579af6 is 50, key is test_row_0/B:col10/1732121554707/Put/seqid=0 2024-11-20T16:52:37,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741955_1131 (size=12301) 2024-11-20T16:52:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T16:52:37,710 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/469b079f936d45528e49c5ece7579af6 2024-11-20T16:52:37,719 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/dbd7c4671ed94099b6484e053153aa58 is 50, key is test_row_0/C:col10/1732121554707/Put/seqid=0 2024-11-20T16:52:37,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741956_1132 (size=12301) 2024-11-20T16:52:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T16:52:38,123 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/dbd7c4671ed94099b6484e053153aa58 2024-11-20T16:52:38,129 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/A/d0beb500515449e7b38344adb3f5aa8e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/d0beb500515449e7b38344adb3f5aa8e 2024-11-20T16:52:38,133 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/d0beb500515449e7b38344adb3f5aa8e, entries=150, sequenceid=627, filesize=12.0 K 2024-11-20T16:52:38,134 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/B/469b079f936d45528e49c5ece7579af6 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/469b079f936d45528e49c5ece7579af6 2024-11-20T16:52:38,138 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/469b079f936d45528e49c5ece7579af6, entries=150, sequenceid=627, filesize=12.0 K 2024-11-20T16:52:38,139 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/.tmp/C/dbd7c4671ed94099b6484e053153aa58 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/dbd7c4671ed94099b6484e053153aa58 2024-11-20T16:52:38,143 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/dbd7c4671ed94099b6484e053153aa58, entries=150, sequenceid=627, filesize=12.0 K 2024-11-20T16:52:38,144 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b683f7af58301e87dbf6f3d58dd983d2 in 1255ms, sequenceid=627, compaction requested=true 2024-11-20T16:52:38,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/b827f961f4a94ce2a2e0c411f838bb1c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/5e4703cddd5843d48341f34f664da336, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/f6d7d400f37940ffab44777e3fcbe617, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/637c05243d014b68867fd386acf7131c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/7c1bbb8a57f045d98a0fb9978587233a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c66fd4c50ea84713b69ed6e34a3c23e8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c5c8c044155d4445896085f27b6c60a8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/faa0434bcf5b401c9e24dad932194890, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/6d6bc545924e4e7db452e03bca86172f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/cf79a39f109d431c8c546c25d4c7b056, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/08121d6a9b6740e18934d9b67a57273a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/3166c3428aea4b8bb6b6c2eb72d385a4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/db0c2d94e98b442386cd9330f1480f24, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/1e06aaf2c04b42a296577ca18d801c0a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/797c708c94e44d69860f7772665ae253, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/14c67f17957140ed9f3d00bbb0cc54ea, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c687973d974642bbbfd5aa0a8c379d77, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/e3695985298e4e7a8f8c6097650e3d67, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/aa89f47dffae45a3b2df91e29e263f43, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c4bd4b602a4a4d48ae6bf39f289060c2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/4a895c1b259f4c1e8df7f8f4605496be, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/70accd13a1e8437f85e272bd7d227a8d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c2b57ae6b02f48b09280ab8022885d99, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/bb902c16302940fca3f1e9e07fca9d6e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/effbc61cd76f44f3a01debd74815075b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/d3144c7965414d75943e1e3f27c95c8e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/e29e43e7ddc24d2cbcf90bbfed010ddf, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/2989b1d7bff449edad2b2f7b4411ce1e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/30dafc9b3b664390bae78a553e931cb6, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/3006f6f4105943209017b68816984f4c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/891ecf8fac3741cca7ebe1ddfeb29ab3, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/16c1690132ed413a929680660a8feab9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/738f2ed0f32f4ec5b9536b73277003dc, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/b5488cbcdd3742409f8bc65ddcb586b9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/37ea35801f504a59aa17365bca96afb9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/13dda130581f4092af37b14e5fee21ae, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/affe1640b02042d8b4004dc4bcef7d5a] to archive 2024-11-20T16:52:38,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T16:52:38,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/b827f961f4a94ce2a2e0c411f838bb1c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/b827f961f4a94ce2a2e0c411f838bb1c 2024-11-20T16:52:38,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/5e4703cddd5843d48341f34f664da336 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/5e4703cddd5843d48341f34f664da336 2024-11-20T16:52:38,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/f6d7d400f37940ffab44777e3fcbe617 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/f6d7d400f37940ffab44777e3fcbe617 2024-11-20T16:52:38,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/637c05243d014b68867fd386acf7131c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/637c05243d014b68867fd386acf7131c 2024-11-20T16:52:38,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/7c1bbb8a57f045d98a0fb9978587233a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/7c1bbb8a57f045d98a0fb9978587233a 2024-11-20T16:52:38,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c66fd4c50ea84713b69ed6e34a3c23e8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c66fd4c50ea84713b69ed6e34a3c23e8 2024-11-20T16:52:38,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c5c8c044155d4445896085f27b6c60a8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c5c8c044155d4445896085f27b6c60a8 2024-11-20T16:52:38,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/faa0434bcf5b401c9e24dad932194890 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/faa0434bcf5b401c9e24dad932194890 2024-11-20T16:52:38,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/6d6bc545924e4e7db452e03bca86172f to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/6d6bc545924e4e7db452e03bca86172f 2024-11-20T16:52:38,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/cf79a39f109d431c8c546c25d4c7b056 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/cf79a39f109d431c8c546c25d4c7b056 2024-11-20T16:52:38,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/08121d6a9b6740e18934d9b67a57273a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/08121d6a9b6740e18934d9b67a57273a 2024-11-20T16:52:38,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/3166c3428aea4b8bb6b6c2eb72d385a4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/3166c3428aea4b8bb6b6c2eb72d385a4 2024-11-20T16:52:38,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/db0c2d94e98b442386cd9330f1480f24 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/db0c2d94e98b442386cd9330f1480f24 2024-11-20T16:52:38,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/1e06aaf2c04b42a296577ca18d801c0a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/1e06aaf2c04b42a296577ca18d801c0a 2024-11-20T16:52:38,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/797c708c94e44d69860f7772665ae253 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/797c708c94e44d69860f7772665ae253 2024-11-20T16:52:38,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/14c67f17957140ed9f3d00bbb0cc54ea to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/14c67f17957140ed9f3d00bbb0cc54ea 2024-11-20T16:52:38,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c687973d974642bbbfd5aa0a8c379d77 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c687973d974642bbbfd5aa0a8c379d77 2024-11-20T16:52:38,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/e3695985298e4e7a8f8c6097650e3d67 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/e3695985298e4e7a8f8c6097650e3d67 2024-11-20T16:52:38,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/aa89f47dffae45a3b2df91e29e263f43 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/aa89f47dffae45a3b2df91e29e263f43 2024-11-20T16:52:38,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c4bd4b602a4a4d48ae6bf39f289060c2 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c4bd4b602a4a4d48ae6bf39f289060c2 2024-11-20T16:52:38,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/4a895c1b259f4c1e8df7f8f4605496be to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/4a895c1b259f4c1e8df7f8f4605496be 2024-11-20T16:52:38,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/70accd13a1e8437f85e272bd7d227a8d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/70accd13a1e8437f85e272bd7d227a8d 2024-11-20T16:52:38,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c2b57ae6b02f48b09280ab8022885d99 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/c2b57ae6b02f48b09280ab8022885d99 2024-11-20T16:52:38,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/bb902c16302940fca3f1e9e07fca9d6e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/bb902c16302940fca3f1e9e07fca9d6e 2024-11-20T16:52:38,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/effbc61cd76f44f3a01debd74815075b to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/effbc61cd76f44f3a01debd74815075b 2024-11-20T16:52:38,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/d3144c7965414d75943e1e3f27c95c8e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/d3144c7965414d75943e1e3f27c95c8e 2024-11-20T16:52:38,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/e29e43e7ddc24d2cbcf90bbfed010ddf to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/e29e43e7ddc24d2cbcf90bbfed010ddf 2024-11-20T16:52:38,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/2989b1d7bff449edad2b2f7b4411ce1e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/2989b1d7bff449edad2b2f7b4411ce1e 2024-11-20T16:52:38,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/30dafc9b3b664390bae78a553e931cb6 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/30dafc9b3b664390bae78a553e931cb6 2024-11-20T16:52:38,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/3006f6f4105943209017b68816984f4c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/3006f6f4105943209017b68816984f4c 2024-11-20T16:52:38,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/891ecf8fac3741cca7ebe1ddfeb29ab3 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/891ecf8fac3741cca7ebe1ddfeb29ab3 2024-11-20T16:52:38,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/16c1690132ed413a929680660a8feab9 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/16c1690132ed413a929680660a8feab9 2024-11-20T16:52:38,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/738f2ed0f32f4ec5b9536b73277003dc to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/738f2ed0f32f4ec5b9536b73277003dc 2024-11-20T16:52:38,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/b5488cbcdd3742409f8bc65ddcb586b9 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/b5488cbcdd3742409f8bc65ddcb586b9 2024-11-20T16:52:38,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/37ea35801f504a59aa17365bca96afb9 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/37ea35801f504a59aa17365bca96afb9 2024-11-20T16:52:38,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/13dda130581f4092af37b14e5fee21ae to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/13dda130581f4092af37b14e5fee21ae 2024-11-20T16:52:38,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/affe1640b02042d8b4004dc4bcef7d5a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/affe1640b02042d8b4004dc4bcef7d5a 2024-11-20T16:52:38,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/ee6f3a44310f4f78ade5979da333ff97, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/4107405e3fce409181668b49994bf3ee, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/085a96f94036497ab14a56a1f592c524, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/fccd21c9c51941c2825790380c1c75d4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/f657202cb7894d158506a72134f0ba71, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/035d7541dae144e9beba984b5da4d96d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/6650bad844384cfd8861d92084a933da, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/f735c844a4c94c28944bc5694aac465c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/c77ef57cd47b4224b91c06a204ddcccf, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/fbb00160643b419abecafcbf30f3ad18, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/c490e436e0d84ff9aee8d03612e27990, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b530438e9a47433f8cdad33a0281015b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b3d843e8389846aa8202c003d8a9d0f9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/7c683e33b1a84ba1ab5151b72928d894, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/4c3ecc43eb564f2cacedb4de4e2ebd85, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/d6cb531401c0459496105b5974a6fe1c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/f0239be5e2064f4ba3652cedb3d1b573, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/82326f9270a145fe9bc7ed425180b639, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/68f75ce3f5fc45cea629d5bdd49fc791, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/ffda9d956a4340b6b4ff3bfcde39660a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/58ea3b9d267b4bef919a46fe630165dc, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/3065b1e1db9b4556ab0ddc09db9e4ed8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b6664a410be248bc956b60c73ac36d49, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/43278fe87017441aaa46327e821868fb, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/04fa8078f0ca4c1dbb67db477a325c2b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/5aa2aff497a24f4c8f65410e48475384, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/a93def6102684a98abebb8202f71d5b1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/d4f3201d375d47008542e7385f8d2ef8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/0edeb2936660444b966c7121e0a9ac94, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/9df0380087184ee7b49738bca66561a2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/5c8afd7cf21f46f9b6ae60e3f9819c82, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/51119fc0e4b0483f9bb0a9f405c27146, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/e12b91e2ee3b41f5b6d959e687d8be6e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/de9177e254f647d3a743058da6e521a7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/d3c5e0fc47fd4bffa23847fa4b580d28, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b656a404646644d6af43286ecdc3c17b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/aa9142eb313f425a8cb94ad5e25ec7b9] to archive 2024-11-20T16:52:38,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T16:52:38,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/ee6f3a44310f4f78ade5979da333ff97 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/ee6f3a44310f4f78ade5979da333ff97 2024-11-20T16:52:38,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/4107405e3fce409181668b49994bf3ee to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/4107405e3fce409181668b49994bf3ee 2024-11-20T16:52:38,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/085a96f94036497ab14a56a1f592c524 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/085a96f94036497ab14a56a1f592c524 2024-11-20T16:52:38,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/fccd21c9c51941c2825790380c1c75d4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/fccd21c9c51941c2825790380c1c75d4 2024-11-20T16:52:38,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/f657202cb7894d158506a72134f0ba71 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/f657202cb7894d158506a72134f0ba71 2024-11-20T16:52:38,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/035d7541dae144e9beba984b5da4d96d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/035d7541dae144e9beba984b5da4d96d 2024-11-20T16:52:38,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/6650bad844384cfd8861d92084a933da to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/6650bad844384cfd8861d92084a933da 2024-11-20T16:52:38,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/f735c844a4c94c28944bc5694aac465c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/f735c844a4c94c28944bc5694aac465c 2024-11-20T16:52:38,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/c77ef57cd47b4224b91c06a204ddcccf to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/c77ef57cd47b4224b91c06a204ddcccf 2024-11-20T16:52:38,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/fbb00160643b419abecafcbf30f3ad18 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/fbb00160643b419abecafcbf30f3ad18 2024-11-20T16:52:38,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/c490e436e0d84ff9aee8d03612e27990 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/c490e436e0d84ff9aee8d03612e27990 2024-11-20T16:52:38,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b530438e9a47433f8cdad33a0281015b to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b530438e9a47433f8cdad33a0281015b 2024-11-20T16:52:38,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b3d843e8389846aa8202c003d8a9d0f9 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b3d843e8389846aa8202c003d8a9d0f9 2024-11-20T16:52:38,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/7c683e33b1a84ba1ab5151b72928d894 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/7c683e33b1a84ba1ab5151b72928d894 2024-11-20T16:52:38,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/4c3ecc43eb564f2cacedb4de4e2ebd85 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/4c3ecc43eb564f2cacedb4de4e2ebd85 2024-11-20T16:52:38,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/d6cb531401c0459496105b5974a6fe1c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/d6cb531401c0459496105b5974a6fe1c 2024-11-20T16:52:38,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/f0239be5e2064f4ba3652cedb3d1b573 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/f0239be5e2064f4ba3652cedb3d1b573 2024-11-20T16:52:38,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/82326f9270a145fe9bc7ed425180b639 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/82326f9270a145fe9bc7ed425180b639 2024-11-20T16:52:38,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/68f75ce3f5fc45cea629d5bdd49fc791 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/68f75ce3f5fc45cea629d5bdd49fc791 2024-11-20T16:52:38,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/ffda9d956a4340b6b4ff3bfcde39660a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/ffda9d956a4340b6b4ff3bfcde39660a 2024-11-20T16:52:38,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/58ea3b9d267b4bef919a46fe630165dc to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/58ea3b9d267b4bef919a46fe630165dc 2024-11-20T16:52:38,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/3065b1e1db9b4556ab0ddc09db9e4ed8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/3065b1e1db9b4556ab0ddc09db9e4ed8 2024-11-20T16:52:38,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b6664a410be248bc956b60c73ac36d49 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b6664a410be248bc956b60c73ac36d49 2024-11-20T16:52:38,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/43278fe87017441aaa46327e821868fb to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/43278fe87017441aaa46327e821868fb 2024-11-20T16:52:38,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/04fa8078f0ca4c1dbb67db477a325c2b to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/04fa8078f0ca4c1dbb67db477a325c2b 2024-11-20T16:52:38,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/5aa2aff497a24f4c8f65410e48475384 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/5aa2aff497a24f4c8f65410e48475384 2024-11-20T16:52:38,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/a93def6102684a98abebb8202f71d5b1 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/a93def6102684a98abebb8202f71d5b1 2024-11-20T16:52:38,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/d4f3201d375d47008542e7385f8d2ef8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/d4f3201d375d47008542e7385f8d2ef8 2024-11-20T16:52:38,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/0edeb2936660444b966c7121e0a9ac94 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/0edeb2936660444b966c7121e0a9ac94 2024-11-20T16:52:38,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/9df0380087184ee7b49738bca66561a2 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/9df0380087184ee7b49738bca66561a2 2024-11-20T16:52:38,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/5c8afd7cf21f46f9b6ae60e3f9819c82 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/5c8afd7cf21f46f9b6ae60e3f9819c82 2024-11-20T16:52:38,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/51119fc0e4b0483f9bb0a9f405c27146 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/51119fc0e4b0483f9bb0a9f405c27146 2024-11-20T16:52:38,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/e12b91e2ee3b41f5b6d959e687d8be6e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/e12b91e2ee3b41f5b6d959e687d8be6e 2024-11-20T16:52:38,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/de9177e254f647d3a743058da6e521a7 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/de9177e254f647d3a743058da6e521a7 2024-11-20T16:52:38,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/d3c5e0fc47fd4bffa23847fa4b580d28 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/d3c5e0fc47fd4bffa23847fa4b580d28 2024-11-20T16:52:38,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b656a404646644d6af43286ecdc3c17b to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/b656a404646644d6af43286ecdc3c17b 2024-11-20T16:52:38,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/aa9142eb313f425a8cb94ad5e25ec7b9 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/aa9142eb313f425a8cb94ad5e25ec7b9 2024-11-20T16:52:38,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/87a636d3f5d5485eb44a35267f73a9ea, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/c3242cdf72644333ba6ca3694d6d5f03, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/bc2a53abcd1844c982828070d6b20647, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/1349769b0f1b4261b1121a45b1e4e867, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/79695afbc5bf42b2b9a405beaa4de4a7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/e5d1dac3a663482f9128c2155ec4818a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/ec87e3b23415451d807cd5cc88978f5a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/9ba956502e874ce78ad5e53e0d1ec472, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/68b9139eef2141258bd5f1c7d6c37d11, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/efd4b4857742492fb06019f49c2a650c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/41470bad2bf843a1a648c1d0563a2bec, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/db91ee01589843fea5e7bcf11efd4efd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/73ba34ab14aa4e169af97189804c40e0, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/e976eee0c8fb4e89befc137aecd6aa9a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/8ed2bd49e45a4fd0ac95029769bdff55, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/27e57b5423e746a2ae7e66fefe2e5edd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3583e366b1d544a3b6fbf220e5c624e1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/52b669afb9684fc5b8a12921697642b9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/42d9585b79e949f7b42dd0f04fdd1d0b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/543eaf97df9c494d99f89cac9118af77, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3f330077b8eb43c680b71e24c0131c21, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/a79c2131b1e445b7b112c6bba4565c0e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/4af45641b61a43a98d71e25bebc78903, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/f74d9f8e580d44b498c4ade3c9229257, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/1fe7c60266234037aa19c8f08d4b5e7a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/fbbdb9a0fd6242bbb4de0887f07ac562, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/74202a971f104e268fe94e3fb0622d51, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/07a0292c19734c45851be953c4ae5810, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/95abed4a85044fd6adf3201bad308e8c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/9bf0ca944e3546fbb10c1835530bc934, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/34bf0deebf194ea2a625b123ffe04ca9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/804ad9f3e20540e89c6f30d3e562e152, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/b3b2dd49595a4e9b8871b5ffd60e2875, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/b32ab5d3e21c4467852581930b671283, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/861fc779c0cc42debd239f91df51b8a4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/fb58f2bedf7d47c2b6da722cd9ef4e31, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3db645fbeada44f386f73ff915913087] to archive 2024-11-20T16:52:38,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T16:52:38,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/87a636d3f5d5485eb44a35267f73a9ea to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/87a636d3f5d5485eb44a35267f73a9ea 2024-11-20T16:52:38,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/c3242cdf72644333ba6ca3694d6d5f03 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/c3242cdf72644333ba6ca3694d6d5f03 2024-11-20T16:52:38,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/bc2a53abcd1844c982828070d6b20647 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/bc2a53abcd1844c982828070d6b20647 2024-11-20T16:52:38,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/1349769b0f1b4261b1121a45b1e4e867 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/1349769b0f1b4261b1121a45b1e4e867 2024-11-20T16:52:38,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/79695afbc5bf42b2b9a405beaa4de4a7 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/79695afbc5bf42b2b9a405beaa4de4a7 2024-11-20T16:52:38,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/e5d1dac3a663482f9128c2155ec4818a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/e5d1dac3a663482f9128c2155ec4818a 2024-11-20T16:52:38,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/ec87e3b23415451d807cd5cc88978f5a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/ec87e3b23415451d807cd5cc88978f5a 2024-11-20T16:52:38,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/9ba956502e874ce78ad5e53e0d1ec472 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/9ba956502e874ce78ad5e53e0d1ec472 2024-11-20T16:52:38,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/68b9139eef2141258bd5f1c7d6c37d11 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/68b9139eef2141258bd5f1c7d6c37d11 2024-11-20T16:52:38,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/efd4b4857742492fb06019f49c2a650c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/efd4b4857742492fb06019f49c2a650c 2024-11-20T16:52:38,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/41470bad2bf843a1a648c1d0563a2bec to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/41470bad2bf843a1a648c1d0563a2bec 2024-11-20T16:52:38,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/db91ee01589843fea5e7bcf11efd4efd to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/db91ee01589843fea5e7bcf11efd4efd 2024-11-20T16:52:38,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/73ba34ab14aa4e169af97189804c40e0 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/73ba34ab14aa4e169af97189804c40e0 2024-11-20T16:52:38,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/e976eee0c8fb4e89befc137aecd6aa9a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/e976eee0c8fb4e89befc137aecd6aa9a 2024-11-20T16:52:38,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/8ed2bd49e45a4fd0ac95029769bdff55 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/8ed2bd49e45a4fd0ac95029769bdff55 2024-11-20T16:52:38,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/27e57b5423e746a2ae7e66fefe2e5edd to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/27e57b5423e746a2ae7e66fefe2e5edd 2024-11-20T16:52:38,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3583e366b1d544a3b6fbf220e5c624e1 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3583e366b1d544a3b6fbf220e5c624e1 2024-11-20T16:52:38,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/52b669afb9684fc5b8a12921697642b9 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/52b669afb9684fc5b8a12921697642b9 2024-11-20T16:52:38,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/42d9585b79e949f7b42dd0f04fdd1d0b to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/42d9585b79e949f7b42dd0f04fdd1d0b 2024-11-20T16:52:38,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/543eaf97df9c494d99f89cac9118af77 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/543eaf97df9c494d99f89cac9118af77 2024-11-20T16:52:38,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3f330077b8eb43c680b71e24c0131c21 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3f330077b8eb43c680b71e24c0131c21 2024-11-20T16:52:38,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/a79c2131b1e445b7b112c6bba4565c0e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/a79c2131b1e445b7b112c6bba4565c0e 2024-11-20T16:52:38,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/4af45641b61a43a98d71e25bebc78903 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/4af45641b61a43a98d71e25bebc78903 2024-11-20T16:52:38,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/f74d9f8e580d44b498c4ade3c9229257 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/f74d9f8e580d44b498c4ade3c9229257 2024-11-20T16:52:38,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/1fe7c60266234037aa19c8f08d4b5e7a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/1fe7c60266234037aa19c8f08d4b5e7a 2024-11-20T16:52:38,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/fbbdb9a0fd6242bbb4de0887f07ac562 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/fbbdb9a0fd6242bbb4de0887f07ac562 2024-11-20T16:52:38,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/74202a971f104e268fe94e3fb0622d51 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/74202a971f104e268fe94e3fb0622d51 2024-11-20T16:52:38,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/07a0292c19734c45851be953c4ae5810 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/07a0292c19734c45851be953c4ae5810 2024-11-20T16:52:38,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/95abed4a85044fd6adf3201bad308e8c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/95abed4a85044fd6adf3201bad308e8c 2024-11-20T16:52:38,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/9bf0ca944e3546fbb10c1835530bc934 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/9bf0ca944e3546fbb10c1835530bc934 2024-11-20T16:52:38,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/34bf0deebf194ea2a625b123ffe04ca9 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/34bf0deebf194ea2a625b123ffe04ca9 2024-11-20T16:52:38,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/804ad9f3e20540e89c6f30d3e562e152 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/804ad9f3e20540e89c6f30d3e562e152 2024-11-20T16:52:38,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/b3b2dd49595a4e9b8871b5ffd60e2875 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/b3b2dd49595a4e9b8871b5ffd60e2875 2024-11-20T16:52:38,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/b32ab5d3e21c4467852581930b671283 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/b32ab5d3e21c4467852581930b671283 2024-11-20T16:52:38,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/861fc779c0cc42debd239f91df51b8a4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/861fc779c0cc42debd239f91df51b8a4 2024-11-20T16:52:38,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/fb58f2bedf7d47c2b6da722cd9ef4e31 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/fb58f2bedf7d47c2b6da722cd9ef4e31 2024-11-20T16:52:38,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3db645fbeada44f386f73ff915913087 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3db645fbeada44f386f73ff915913087 2024-11-20T16:52:38,298 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/recovered.edits/630.seqid, newMaxSeqId=630, maxSeqId=1 2024-11-20T16:52:38,298 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. 2024-11-20T16:52:38,299 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for b683f7af58301e87dbf6f3d58dd983d2: Waiting for close lock at 1732121556889Running coprocessor pre-close hooks at 1732121556889Disabling compacts and flushes for region at 1732121556889Disabling writes for close at 1732121556889Obtaining lock to block concurrent updates at 1732121556889Preparing flush snapshotting stores in b683f7af58301e87dbf6f3d58dd983d2 at 1732121556889Finished memstore snapshotting TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732121556890 (+1 ms)Flushing stores of TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2. at 1732121556890Flushing b683f7af58301e87dbf6f3d58dd983d2/A: creating writer at 1732121556890Flushing b683f7af58301e87dbf6f3d58dd983d2/A: appending metadata at 1732121556893 (+3 ms)Flushing b683f7af58301e87dbf6f3d58dd983d2/A: closing flushed file at 1732121556893Flushing b683f7af58301e87dbf6f3d58dd983d2/B: creating writer at 1732121557302 (+409 ms)Flushing b683f7af58301e87dbf6f3d58dd983d2/B: appending metadata at 1732121557305 (+3 ms)Flushing b683f7af58301e87dbf6f3d58dd983d2/B: closing flushed file at 1732121557305Flushing b683f7af58301e87dbf6f3d58dd983d2/C: creating writer at 1732121557715 (+410 ms)Flushing b683f7af58301e87dbf6f3d58dd983d2/C: appending metadata at 1732121557718 (+3 ms)Flushing b683f7af58301e87dbf6f3d58dd983d2/C: closing flushed file at 1732121557718Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1feb3593: reopening flushed file at 1732121558128 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77907f6f: reopening flushed file at 1732121558133 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f51b125: reopening flushed file at 1732121558138 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b683f7af58301e87dbf6f3d58dd983d2 in 1255ms, sequenceid=627, compaction requested=true at 1732121558144 (+6 ms)Writing region close event to WAL at 1732121558294 (+150 ms)Running coprocessor post-close hooks at 1732121558298 (+4 ms)Closed at 1732121558298 2024-11-20T16:52:38,301 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:38,301 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=b683f7af58301e87dbf6f3d58dd983d2, regionState=CLOSED 2024-11-20T16:52:38,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 because future has completed 2024-11-20T16:52:38,306 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-20T16:52:38,306 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure b683f7af58301e87dbf6f3d58dd983d2, server=b60eac9bda43,33929,1732121526384 in 1.5720 sec 2024-11-20T16:52:38,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-20T16:52:38,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b683f7af58301e87dbf6f3d58dd983d2, UNASSIGN in 1.5800 sec 2024-11-20T16:52:38,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-20T16:52:38,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5880 sec 2024-11-20T16:52:38,314 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121558313"}]},"ts":"1732121558313"} 2024-11-20T16:52:38,316 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T16:52:38,316 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T16:52:38,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6050 sec 2024-11-20T16:52:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T16:52:38,876 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:38,879 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T16:52:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:52:38,883 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:52:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-20T16:52:38,884 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:52:38,889 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:38,892 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A, FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B, FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C, FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/recovered.edits] 2024-11-20T16:52:38,895 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/5d2fff634d4645ea8439836429583669 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/5d2fff634d4645ea8439836429583669 2024-11-20T16:52:38,896 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/d0beb500515449e7b38344adb3f5aa8e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/d0beb500515449e7b38344adb3f5aa8e 2024-11-20T16:52:38,897 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/f30e55d05d804586b57db10d2f072dab to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/A/f30e55d05d804586b57db10d2f072dab 2024-11-20T16:52:38,899 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/3e4b5cf68f634baaba9d4bf8076a881a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/3e4b5cf68f634baaba9d4bf8076a881a 2024-11-20T16:52:38,900 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/469b079f936d45528e49c5ece7579af6 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/469b079f936d45528e49c5ece7579af6 2024-11-20T16:52:38,902 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/a56b65c468d34acdb7e9a709a52bf260 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/B/a56b65c468d34acdb7e9a709a52bf260 2024-11-20T16:52:38,904 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3c6848464d594c0f88775f1da29f9b8e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/3c6848464d594c0f88775f1da29f9b8e 2024-11-20T16:52:38,906 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/84ccd4d5e86c4acc950e6e8f567458ed to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/84ccd4d5e86c4acc950e6e8f567458ed 2024-11-20T16:52:38,907 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/dbd7c4671ed94099b6484e053153aa58 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/C/dbd7c4671ed94099b6484e053153aa58 2024-11-20T16:52:38,909 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/recovered.edits/630.seqid to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2/recovered.edits/630.seqid 2024-11-20T16:52:38,910 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/b683f7af58301e87dbf6f3d58dd983d2 2024-11-20T16:52:38,910 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T16:52:38,915 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:52:38,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-20T16:52:38,925 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T16:52:38,953 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T16:52:38,954 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:52:38,954 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T16:52:38,954 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732121558954"}]},"ts":"9223372036854775807"} 2024-11-20T16:52:38,959 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T16:52:38,959 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b683f7af58301e87dbf6f3d58dd983d2, NAME => 'TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T16:52:38,959 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T16:52:38,959 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732121558959"}]},"ts":"9223372036854775807"} 2024-11-20T16:52:38,963 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T16:52:38,964 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:52:38,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 85 msec 2024-11-20T16:52:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-20T16:52:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T16:52:38,997 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:39,010 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-48111169_22 at /127.0.0.1:41550 [Waiting for operation #381] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_960049648_22 at /127.0.0.1:54838 [Waiting for operation #361] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;b60eac9bda43:33929-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=402 (was 311) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6605 (was 7365) 2024-11-20T16:52:39,022 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=402, ProcessCount=11, AvailableMemoryMB=6603 2024-11-20T16:52:39,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T16:52:39,025 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T16:52:39,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T16:52:39,027 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T16:52:39,028 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:39,028 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-20T16:52:39,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-20T16:52:39,029 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T16:52:39,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741957_1133 (size=1061) 2024-11-20T16:52:39,038 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625 2024-11-20T16:52:39,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741958_1134 (size=53) 2024-11-20T16:52:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-20T16:52:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-20T16:52:39,445 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:52:39,445 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 94c122f231a59b575ec7f40dae2c00f4, disabling compactions & flushes 2024-11-20T16:52:39,445 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:39,445 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:39,445 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. after waiting 0 ms 2024-11-20T16:52:39,445 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:39,445 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:39,445 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 94c122f231a59b575ec7f40dae2c00f4: Waiting for close lock at 1732121559445Disabling compacts and flushes for region at 1732121559445Disabling writes for close at 1732121559445Writing region close event to WAL at 1732121559445Closed at 1732121559445 2024-11-20T16:52:39,446 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T16:52:39,447 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732121559446"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732121559446"}]},"ts":"1732121559446"} 2024-11-20T16:52:39,449 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T16:52:39,449 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T16:52:39,450 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121559449"}]},"ts":"1732121559449"} 2024-11-20T16:52:39,451 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T16:52:39,451 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c122f231a59b575ec7f40dae2c00f4, ASSIGN}] 2024-11-20T16:52:39,453 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c122f231a59b575ec7f40dae2c00f4, ASSIGN 2024-11-20T16:52:39,453 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c122f231a59b575ec7f40dae2c00f4, ASSIGN; state=OFFLINE, location=b60eac9bda43,33929,1732121526384; forceNewPlan=false, retain=false 2024-11-20T16:52:39,604 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=94c122f231a59b575ec7f40dae2c00f4, regionState=OPENING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:39,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c122f231a59b575ec7f40dae2c00f4, ASSIGN because future has completed 2024-11-20T16:52:39,607 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:52:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-20T16:52:39,762 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:39,762 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} 2024-11-20T16:52:39,762 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:39,762 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:52:39,762 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:39,762 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:39,764 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:39,765 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:52:39,766 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c122f231a59b575ec7f40dae2c00f4 columnFamilyName A 2024-11-20T16:52:39,766 DEBUG [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:39,766 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] regionserver.HStore(327): Store=94c122f231a59b575ec7f40dae2c00f4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:52:39,766 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:39,767 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:52:39,768 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c122f231a59b575ec7f40dae2c00f4 columnFamilyName B 2024-11-20T16:52:39,768 DEBUG [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:39,768 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] regionserver.HStore(327): Store=94c122f231a59b575ec7f40dae2c00f4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:52:39,768 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:39,769 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:52:39,769 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c122f231a59b575ec7f40dae2c00f4 columnFamilyName C 2024-11-20T16:52:39,769 DEBUG [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:39,770 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] regionserver.HStore(327): Store=94c122f231a59b575ec7f40dae2c00f4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:52:39,770 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:39,770 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:39,771 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:39,771 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:39,772 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:39,772 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:39,773 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T16:52:39,774 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:39,776 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T16:52:39,776 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 94c122f231a59b575ec7f40dae2c00f4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58812214, jitterRate=-0.12362971901893616}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T16:52:39,776 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:39,777 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 94c122f231a59b575ec7f40dae2c00f4: Running coprocessor pre-open hook at 1732121559763Writing region info on filesystem at 1732121559763Initializing all the Stores at 1732121559763Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121559763Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121559764 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121559764Cleaning up temporary data from old regions at 1732121559772 (+8 ms)Running coprocessor post-open hooks at 1732121559776 (+4 ms)Region opened successfully at 1732121559777 (+1 ms) 2024-11-20T16:52:39,778 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., pid=36, masterSystemTime=1732121559759 2024-11-20T16:52:39,780 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:39,780 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:39,780 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=94c122f231a59b575ec7f40dae2c00f4, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:39,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 because future has completed 2024-11-20T16:52:39,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-20T16:52:39,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 in 176 msec 2024-11-20T16:52:39,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-20T16:52:39,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c122f231a59b575ec7f40dae2c00f4, ASSIGN in 333 msec 2024-11-20T16:52:39,787 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T16:52:39,787 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121559787"}]},"ts":"1732121559787"} 2024-11-20T16:52:39,788 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T16:52:39,789 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T16:52:39,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 764 msec 2024-11-20T16:52:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-20T16:52:40,166 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:40,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:40,167 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:40,168 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:40,169 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78d5d3a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:40,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:40,171 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:40,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cedc1ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:40,172 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:40,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:40,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:40,175 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T16:52:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is b60eac9bda43,33377,1732121525625 2024-11-20T16:52:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@17eb0dc9 2024-11-20T16:52:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T16:52:40,177 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T16:52:40,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T16:52:40,181 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T16:52:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T16:52:40,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741959_1135 (size=1097) 2024-11-20T16:52:40,600 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-20T16:52:40,600 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-20T16:52:40,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T16:52:40,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c122f231a59b575ec7f40dae2c00f4, REOPEN/MOVE}] 2024-11-20T16:52:40,611 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c122f231a59b575ec7f40dae2c00f4, REOPEN/MOVE 2024-11-20T16:52:40,612 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=94c122f231a59b575ec7f40dae2c00f4, regionState=CLOSING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:40,615 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c122f231a59b575ec7f40dae2c00f4, REOPEN/MOVE because future has completed 2024-11-20T16:52:40,615 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T16:52:40,615 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:52:40,768 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:40,768 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T16:52:40,768 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 94c122f231a59b575ec7f40dae2c00f4, disabling compactions & flushes 2024-11-20T16:52:40,768 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:40,768 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:40,768 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. after waiting 0 ms 2024-11-20T16:52:40,768 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:40,772 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T16:52:40,773 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:40,773 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 94c122f231a59b575ec7f40dae2c00f4: Waiting for close lock at 1732121560768Running coprocessor pre-close hooks at 1732121560768Disabling compacts and flushes for region at 1732121560768Disabling writes for close at 1732121560768Writing region close event to WAL at 1732121560769 (+1 ms)Running coprocessor post-close hooks at 1732121560773 (+4 ms)Closed at 1732121560773 2024-11-20T16:52:40,773 WARN [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 94c122f231a59b575ec7f40dae2c00f4 to self. 2024-11-20T16:52:40,775 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:40,776 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=94c122f231a59b575ec7f40dae2c00f4, regionState=CLOSED 2024-11-20T16:52:40,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 because future has completed 2024-11-20T16:52:40,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-20T16:52:40,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 in 163 msec 2024-11-20T16:52:40,781 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c122f231a59b575ec7f40dae2c00f4, REOPEN/MOVE; state=CLOSED, location=b60eac9bda43,33929,1732121526384; forceNewPlan=false, retain=true 2024-11-20T16:52:40,931 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=94c122f231a59b575ec7f40dae2c00f4, regionState=OPENING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:40,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c122f231a59b575ec7f40dae2c00f4, REOPEN/MOVE because future has completed 2024-11-20T16:52:40,933 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:52:41,088 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:41,088 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} 2024-11-20T16:52:41,089 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:41,089 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:52:41,089 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:41,089 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:41,091 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:41,092 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:52:41,095 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c122f231a59b575ec7f40dae2c00f4 columnFamilyName A 2024-11-20T16:52:41,097 DEBUG [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:41,097 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] regionserver.HStore(327): Store=94c122f231a59b575ec7f40dae2c00f4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:52:41,098 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:41,098 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:52:41,098 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c122f231a59b575ec7f40dae2c00f4 columnFamilyName B 2024-11-20T16:52:41,099 DEBUG [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:41,099 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] regionserver.HStore(327): Store=94c122f231a59b575ec7f40dae2c00f4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:52:41,099 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:41,100 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:52:41,100 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c122f231a59b575ec7f40dae2c00f4 columnFamilyName C 2024-11-20T16:52:41,100 DEBUG [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:41,100 INFO [StoreOpener-94c122f231a59b575ec7f40dae2c00f4-1 {}] regionserver.HStore(327): Store=94c122f231a59b575ec7f40dae2c00f4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:52:41,101 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:41,101 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:41,101 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:41,102 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:41,104 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:41,104 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:41,104 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T16:52:41,106 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:41,106 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 94c122f231a59b575ec7f40dae2c00f4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73984601, jitterRate=0.1024564653635025}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T16:52:41,106 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:41,108 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 94c122f231a59b575ec7f40dae2c00f4: Running coprocessor pre-open hook at 1732121561089Writing region info on filesystem at 1732121561089Initializing all the Stores at 1732121561090 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121561090Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121561090Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121561090Cleaning up temporary data from old regions at 1732121561104 (+14 ms)Running coprocessor post-open hooks at 1732121561107 (+3 ms)Region opened successfully at 1732121561108 (+1 ms) 2024-11-20T16:52:41,108 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., pid=41, masterSystemTime=1732121561085 2024-11-20T16:52:41,110 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:41,110 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:41,110 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=94c122f231a59b575ec7f40dae2c00f4, regionState=OPEN, openSeqNum=5, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 because future has completed 2024-11-20T16:52:41,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-20T16:52:41,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 in 180 msec 2024-11-20T16:52:41,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-20T16:52:41,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c122f231a59b575ec7f40dae2c00f4, REOPEN/MOVE in 505 msec 2024-11-20T16:52:41,120 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-20T16:52:41,120 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 515 msec 2024-11-20T16:52:41,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 938 msec 2024-11-20T16:52:41,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-20T16:52:41,126 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:41,126 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,126 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:41,126 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:41,127 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b33a7f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:41,129 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:41,130 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:41,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d89034, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,131 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:41,131 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:41,132 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:41,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5efd2437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:41,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:41,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:41,134 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:41,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35ce6d06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,136 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:41,136 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:41,137 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@475588b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:41,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:41,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:41,139 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:41,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8fa9400, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,141 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:41,141 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:41,142 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d1f1be2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:41,143 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:41,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33b1dda4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,145 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:41,145 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:41,146 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:41,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:41,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:41,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@521dc994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:41,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:41,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:41,147 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0a9a5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,149 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:41,149 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:41,149 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62af7d44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:41,151 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47fcac80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,152 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,152 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:41,152 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:41,153 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55f1c931, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:41,154 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7203a827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,155 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:41,155 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:41,156 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bfb8bf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:41,157 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7825eb66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,158 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:52:41,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:52:41,159 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:52:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:52:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:52:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cd11e96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:52:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:52:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:52:41,160 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:52:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411e8aaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:52:41,161 DEBUG [Thread-674 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:41,161 DEBUG [Thread-681 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:41,162 DEBUG [Thread-687 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:41,162 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:41,163 DEBUG [Thread-672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:41,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:41,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:41,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-20T16:52:41,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:41,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:41,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:41,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:41,164 DEBUG [Thread-683 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:41,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:41,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:41,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T16:52:41,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:41,165 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:41,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:41,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:41,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:41,166 DEBUG [Thread-676 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:41,166 DEBUG [Thread-670 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:41,166 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:41,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:41,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:41,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:41,167 DEBUG [Thread-685 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:41,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:52:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:52:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:52:41,172 DEBUG [Thread-678 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:52:41,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:41,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:41,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:52:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:52:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:52:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:52:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:52:41,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:52:41,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:52:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:52:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:52:41,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T16:52:41,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:41,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:41,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:41,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:41,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:41,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:41,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:41,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206c3c3442187f4451aec25bfa2b347c59_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121561194/Put/seqid=0 2024-11-20T16:52:41,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741960_1136 (size=12154) 2024-11-20T16:52:41,250 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:41,258 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206c3c3442187f4451aec25bfa2b347c59_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206c3c3442187f4451aec25bfa2b347c59_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:41,260 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/8765d4ce8d884ce4af94ba8e5561d865, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:41,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45236 deadline: 1732121621248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121621252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121621254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45304 deadline: 1732121621255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/8765d4ce8d884ce4af94ba8e5561d865 is 175, key is test_row_0/A:col10/1732121561194/Put/seqid=0 2024-11-20T16:52:41,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121621265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T16:52:41,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T16:52:41,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:41,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:41,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:41,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:41,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:41,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741961_1137 (size=30955) 2024-11-20T16:52:41,332 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/8765d4ce8d884ce4af94ba8e5561d865 2024-11-20T16:52:41,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/cd4d17870ea64306967e2966a9e55cae is 50, key is test_row_0/B:col10/1732121561194/Put/seqid=0 2024-11-20T16:52:41,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45304 deadline: 1732121621376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121621377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121621377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45236 deadline: 1732121621378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121621387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741962_1138 (size=12001) 2024-11-20T16:52:41,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/cd4d17870ea64306967e2966a9e55cae 2024-11-20T16:52:41,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/cd2cc91b51e247ebbf2b725957e2eb7c is 50, key is test_row_0/C:col10/1732121561194/Put/seqid=0 2024-11-20T16:52:41,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T16:52:41,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:41,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:41,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:41,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:41,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:41,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741963_1139 (size=12001) 2024-11-20T16:52:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T16:52:41,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/cd2cc91b51e247ebbf2b725957e2eb7c 2024-11-20T16:52:41,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/8765d4ce8d884ce4af94ba8e5561d865 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/8765d4ce8d884ce4af94ba8e5561d865 2024-11-20T16:52:41,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/8765d4ce8d884ce4af94ba8e5561d865, entries=150, sequenceid=16, filesize=30.2 K 2024-11-20T16:52:41,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/cd4d17870ea64306967e2966a9e55cae as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/cd4d17870ea64306967e2966a9e55cae 2024-11-20T16:52:41,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/cd4d17870ea64306967e2966a9e55cae, entries=150, sequenceid=16, filesize=11.7 K 2024-11-20T16:52:41,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/cd2cc91b51e247ebbf2b725957e2eb7c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/cd2cc91b51e247ebbf2b725957e2eb7c 2024-11-20T16:52:41,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/cd2cc91b51e247ebbf2b725957e2eb7c, entries=150, sequenceid=16, filesize=11.7 K 2024-11-20T16:52:41,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 94c122f231a59b575ec7f40dae2c00f4 in 336ms, sequenceid=16, compaction requested=false 2024-11-20T16:52:41,533 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T16:52:41,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:41,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121561244, max timestamp=1732121561587Num uniques 859; 2024-11-20T16:52:41,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:52:41,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121561244, max timestamp=1732121561587Num uniques 859; 2024-11-20T16:52:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:41,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121561244, max timestamp=1732121561587Num uniques 859; 2024-11-20T16:52:41,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:52:41,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:52:41,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=40.25 KB heapSize=106.08 KB 2024-11-20T16:52:41,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:41,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:41,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:41,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:41,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:41,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:41,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208eff79d3b765418c954795db590f5438_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121561262/Put/seqid=0 2024-11-20T16:52:41,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741964_1140 (size=12154) 2024-11-20T16:52:41,625 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:41,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T16:52:41,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:41,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:41,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:41,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:41,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:41,635 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208eff79d3b765418c954795db590f5438_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208eff79d3b765418c954795db590f5438_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:41,637 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/f00e8aad3f594453a622ba28c883ee27, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:41,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/f00e8aad3f594453a622ba28c883ee27 is 175, key is test_row_0/A:col10/1732121561262/Put/seqid=0 2024-11-20T16:52:41,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732121561636, max timestamp=1732121561652Num uniques 859; 2024-11-20T16:52:41,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T16:52:41,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732121561636, max timestamp=1732121561653Num uniques 859; 2024-11-20T16:52:41,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T16:52:41,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732121561636, max timestamp=1732121561652Num uniques 859; 2024-11-20T16:52:41,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T16:52:41,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741965_1141 (size=30955) 2024-11-20T16:52:41,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121621686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121621692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45304 deadline: 1732121621695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121621697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45236 deadline: 1732121621699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T16:52:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:41,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:41,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:41,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:41,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T16:52:41,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121621808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121621809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121621816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45236 deadline: 1732121621816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45304 deadline: 1732121621816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:41,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:41,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:41,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T16:52:41,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:41,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:41,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:41,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:41,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:42,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121622016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121622016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121622026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45236 deadline: 1732121622026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45304 deadline: 1732121622026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,067 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=39, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/f00e8aad3f594453a622ba28c883ee27 2024-11-20T16:52:42,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/f465769f091b44e6865641ea416beed7 is 50, key is test_row_0/B:col10/1732121561262/Put/seqid=0 2024-11-20T16:52:42,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741966_1142 (size=12001) 2024-11-20T16:52:42,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/f465769f091b44e6865641ea416beed7 2024-11-20T16:52:42,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T16:52:42,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:42,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:42,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:42,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:42,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:42,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/dda406cb62594dff9d41e7b4dffbe34d is 50, key is test_row_0/C:col10/1732121561262/Put/seqid=0 2024-11-20T16:52:42,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741967_1143 (size=12001) 2024-11-20T16:52:42,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/dda406cb62594dff9d41e7b4dffbe34d 2024-11-20T16:52:42,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/f00e8aad3f594453a622ba28c883ee27 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f00e8aad3f594453a622ba28c883ee27 2024-11-20T16:52:42,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f00e8aad3f594453a622ba28c883ee27, entries=150, sequenceid=39, filesize=30.2 K 2024-11-20T16:52:42,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/f465769f091b44e6865641ea416beed7 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/f465769f091b44e6865641ea416beed7 2024-11-20T16:52:42,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/f465769f091b44e6865641ea416beed7, entries=150, sequenceid=39, filesize=11.7 K 2024-11-20T16:52:42,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/dda406cb62594dff9d41e7b4dffbe34d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/dda406cb62594dff9d41e7b4dffbe34d 2024-11-20T16:52:42,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/dda406cb62594dff9d41e7b4dffbe34d, entries=150, sequenceid=39, filesize=11.7 K 2024-11-20T16:52:42,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=147.60 KB/151140 for 94c122f231a59b575ec7f40dae2c00f4 in 587ms, sequenceid=39, compaction requested=false 2024-11-20T16:52:42,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:42,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T16:52:42,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:42,253 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-20T16:52:42,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:42,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:42,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:42,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:42,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:42,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:42,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200bbc3423cc1a469681c5952366c01155_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121561694/Put/seqid=0 2024-11-20T16:52:42,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741968_1144 (size=12154) 2024-11-20T16:52:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T16:52:42,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:42,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:42,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45236 deadline: 1732121622337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45304 deadline: 1732121622337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121622338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121622343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121622345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121622456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121622456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121622456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121622667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121622667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121622667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:42,688 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200bbc3423cc1a469681c5952366c01155_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200bbc3423cc1a469681c5952366c01155_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:42,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/c81730be4a5f41b99877ac8a7f5ad622, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:42,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/c81730be4a5f41b99877ac8a7f5ad622 is 175, key is test_row_0/A:col10/1732121561694/Put/seqid=0 2024-11-20T16:52:42,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741969_1145 (size=30955) 2024-11-20T16:52:42,705 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=75, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/c81730be4a5f41b99877ac8a7f5ad622 2024-11-20T16:52:42,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/94f2bd42818e4b89b48d1ecea55c5c6c is 50, key is test_row_0/B:col10/1732121561694/Put/seqid=0 2024-11-20T16:52:42,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741970_1146 (size=12001) 2024-11-20T16:52:42,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45236 deadline: 1732121622856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45304 deadline: 1732121622856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121622976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121622976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:42,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121622985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:43,147 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/94f2bd42818e4b89b48d1ecea55c5c6c 2024-11-20T16:52:43,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/8e737d7a89ed467d86382c1af0801e0a is 50, key is test_row_0/C:col10/1732121561694/Put/seqid=0 2024-11-20T16:52:43,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741971_1147 (size=12001) 2024-11-20T16:52:43,168 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/8e737d7a89ed467d86382c1af0801e0a 2024-11-20T16:52:43,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/c81730be4a5f41b99877ac8a7f5ad622 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/c81730be4a5f41b99877ac8a7f5ad622 2024-11-20T16:52:43,177 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/c81730be4a5f41b99877ac8a7f5ad622, entries=150, sequenceid=75, filesize=30.2 K 2024-11-20T16:52:43,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/94f2bd42818e4b89b48d1ecea55c5c6c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/94f2bd42818e4b89b48d1ecea55c5c6c 2024-11-20T16:52:43,184 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/94f2bd42818e4b89b48d1ecea55c5c6c, entries=150, sequenceid=75, filesize=11.7 K 2024-11-20T16:52:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/8e737d7a89ed467d86382c1af0801e0a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/8e737d7a89ed467d86382c1af0801e0a 2024-11-20T16:52:43,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,203 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/8e737d7a89ed467d86382c1af0801e0a, entries=150, sequenceid=75, filesize=11.7 K 2024-11-20T16:52:43,204 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 94c122f231a59b575ec7f40dae2c00f4 in 951ms, sequenceid=75, compaction requested=true 2024-11-20T16:52:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:43,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:43,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-20T16:52:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-20T16:52:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-20T16:52:43,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0400 sec 2024-11-20T16:52:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.0480 sec 2024-11-20T16:52:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T16:52:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,327 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:43,327 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:43,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-20T16:52:43,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T16:52:43,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,332 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:43,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,333 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:43,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T16:52:43,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T16:52:43,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:43,487 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-20T16:52:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:43,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:43,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:43,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:43,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:43,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209aac1ba1c3194e61a9fcaecc3b2c4fee_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121562342/Put/seqid=0 2024-11-20T16:52:43,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741972_1148 (size=9714) 2024-11-20T16:52:43,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,534 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209aac1ba1c3194e61a9fcaecc3b2c4fee_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209aac1ba1c3194e61a9fcaecc3b2c4fee_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:43,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/d7689b73f8b64156955240884e503df5, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:43,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/d7689b73f8b64156955240884e503df5 is 175, key is test_row_0/A:col10/1732121562342/Put/seqid=0 2024-11-20T16:52:43,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121563512, max timestamp=1732121563552Num uniques 859; 2024-11-20T16:52:43,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121563512, max timestamp=1732121563552Num uniques 859; 2024-11-20T16:52:43,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121563512, max timestamp=1732121563552Num uniques 859; 2024-11-20T16:52:43,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:52:43,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:52:43,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:52:43,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741973_1149 (size=22361) 2024-11-20T16:52:43,567 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/d7689b73f8b64156955240884e503df5 2024-11-20T16:52:43,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/d5ad19dd112a430cb1ac3695bf2907df is 50, key is test_row_0/B:col10/1732121562342/Put/seqid=0 2024-11-20T16:52:43,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:43,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121623606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:43,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121623611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:43,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732121563601, max timestamp=1732121563611Num uniques 1018; 2024-11-20T16:52:43,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-20T16:52:43,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732121563601, max timestamp=1732121563611Num uniques 1018; 2024-11-20T16:52:43,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-20T16:52:43,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741974_1150 (size=9657) 2024-11-20T16:52:43,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732121563601, max timestamp=1732121563611Num uniques 1018; 2024-11-20T16:52:43,622 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/d5ad19dd112a430cb1ac3695bf2907df 2024-11-20T16:52:43,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-20T16:52:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T16:52:43,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/3058e11aa4b4441d822d46e501fc538e is 50, key is test_row_0/C:col10/1732121562342/Put/seqid=0 2024-11-20T16:52:43,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741975_1151 (size=9657) 2024-11-20T16:52:43,658 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/3058e11aa4b4441d822d46e501fc538e 2024-11-20T16:52:43,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/d7689b73f8b64156955240884e503df5 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/d7689b73f8b64156955240884e503df5 2024-11-20T16:52:43,671 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/d7689b73f8b64156955240884e503df5, entries=100, sequenceid=84, filesize=21.8 K 2024-11-20T16:52:43,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/d5ad19dd112a430cb1ac3695bf2907df as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/d5ad19dd112a430cb1ac3695bf2907df 2024-11-20T16:52:43,679 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/d5ad19dd112a430cb1ac3695bf2907df, entries=100, sequenceid=84, filesize=9.4 K 2024-11-20T16:52:43,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/3058e11aa4b4441d822d46e501fc538e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/3058e11aa4b4441d822d46e501fc538e 2024-11-20T16:52:43,687 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/3058e11aa4b4441d822d46e501fc538e, entries=100, sequenceid=84, filesize=9.4 K 2024-11-20T16:52:43,691 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=87.22 KB/89310 for 94c122f231a59b575ec7f40dae2c00f4 in 203ms, sequenceid=84, compaction requested=true 2024-11-20T16:52:43,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:43,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:43,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-20T16:52:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-20T16:52:43,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-20T16:52:43,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 359 msec 2024-11-20T16:52:43,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 366 msec 2024-11-20T16:52:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:43,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=93.93 KB heapSize=259.92 KB 2024-11-20T16:52:43,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:43,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:43,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:43,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:43,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:43,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:43,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c41b6dc77e1c487aa86d2556f1f5d303_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121563697/Put/seqid=0 2024-11-20T16:52:43,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741976_1152 (size=14694) 2024-11-20T16:52:43,734 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:43,748 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c41b6dc77e1c487aa86d2556f1f5d303_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c41b6dc77e1c487aa86d2556f1f5d303_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:43,749 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/2cf44e08c23b42ab93bb51652adc8b8f, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:43,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/2cf44e08c23b42ab93bb51652adc8b8f is 175, key is test_row_0/A:col10/1732121563697/Put/seqid=0 2024-11-20T16:52:43,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741977_1153 (size=39649) 2024-11-20T16:52:43,768 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/2cf44e08c23b42ab93bb51652adc8b8f 2024-11-20T16:52:43,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121623754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:43,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121623756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:43,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:43,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121623757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:43,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:43,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/3fd67cbf28c249d4a19291840a381364 is 50, key is test_row_0/B:col10/1732121563697/Put/seqid=0 2024-11-20T16:52:43,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741978_1154 (size=12051) 2024-11-20T16:52:43,853 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T16:52:43,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45236 deadline: 1732121623877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:43,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45304 deadline: 1732121623877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:43,888 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 94c122f231a59b575ec7f40dae2c00f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:52:43,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,888 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 94c122f231a59b575ec7f40dae2c00f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:52:43,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:43,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:43,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121623877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:43,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121623878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:43,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121623895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:43,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T16:52:43,956 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:43,956 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-20T16:52:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T16:52:43,958 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:43,959 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:43,959 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T16:52:44,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121624096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:44,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121624096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:44,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121624106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:44,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:44,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T16:52:44,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:44,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:44,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:44,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:44,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:44,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/3fd67cbf28c249d4a19291840a381364 2024-11-20T16:52:44,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/aa834389a15d4822bf6fbaa2bc93d800 is 50, key is test_row_0/C:col10/1732121563697/Put/seqid=0 2024-11-20T16:52:44,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741979_1155 (size=12051) 2024-11-20T16:52:44,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/aa834389a15d4822bf6fbaa2bc93d800 2024-11-20T16:52:44,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T16:52:44,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:44,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:44,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:44,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:44,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T16:52:44,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/2cf44e08c23b42ab93bb51652adc8b8f as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/2cf44e08c23b42ab93bb51652adc8b8f 2024-11-20T16:52:44,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/2cf44e08c23b42ab93bb51652adc8b8f, entries=200, sequenceid=131, filesize=38.7 K 2024-11-20T16:52:44,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/3fd67cbf28c249d4a19291840a381364 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/3fd67cbf28c249d4a19291840a381364 2024-11-20T16:52:44,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/3fd67cbf28c249d4a19291840a381364, entries=150, sequenceid=131, filesize=11.8 K 2024-11-20T16:52:44,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/aa834389a15d4822bf6fbaa2bc93d800 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/aa834389a15d4822bf6fbaa2bc93d800 2024-11-20T16:52:44,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/aa834389a15d4822bf6fbaa2bc93d800, entries=150, sequenceid=131, filesize=11.8 K 2024-11-20T16:52:44,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~259.88 KB/266112, currentSize=93.93 KB/96180 for 94c122f231a59b575ec7f40dae2c00f4 in 611ms, sequenceid=131, compaction requested=true 2024-11-20T16:52:44,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:44,311 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T16:52:44,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:44,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:44,312 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T16:52:44,312 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 154875 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T16:52:44,313 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/A is initiating minor compaction (all files) 2024-11-20T16:52:44,313 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/A in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:44,313 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/8765d4ce8d884ce4af94ba8e5561d865, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f00e8aad3f594453a622ba28c883ee27, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/c81730be4a5f41b99877ac8a7f5ad622, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/d7689b73f8b64156955240884e503df5, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/2cf44e08c23b42ab93bb51652adc8b8f] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=151.2 K 2024-11-20T16:52:44,313 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:44,313 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/8765d4ce8d884ce4af94ba8e5561d865, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f00e8aad3f594453a622ba28c883ee27, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/c81730be4a5f41b99877ac8a7f5ad622, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/d7689b73f8b64156955240884e503df5, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/2cf44e08c23b42ab93bb51652adc8b8f] 2024-11-20T16:52:44,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:44,314 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8765d4ce8d884ce4af94ba8e5561d865, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732121561193 2024-11-20T16:52:44,315 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting f00e8aad3f594453a622ba28c883ee27, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732121561262 2024-11-20T16:52:44,316 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57711 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T16:52:44,316 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/B is initiating minor compaction (all files) 2024-11-20T16:52:44,316 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/B in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:44,316 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/cd4d17870ea64306967e2966a9e55cae, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/f465769f091b44e6865641ea416beed7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/94f2bd42818e4b89b48d1ecea55c5c6c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/d5ad19dd112a430cb1ac3695bf2907df, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/3fd67cbf28c249d4a19291840a381364] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=56.4 K 2024-11-20T16:52:44,317 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting cd4d17870ea64306967e2966a9e55cae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732121561193 2024-11-20T16:52:44,317 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting c81730be4a5f41b99877ac8a7f5ad622, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732121561683 2024-11-20T16:52:44,317 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting f465769f091b44e6865641ea416beed7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732121561262 2024-11-20T16:52:44,318 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7689b73f8b64156955240884e503df5, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732121562335 2024-11-20T16:52:44,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:44,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:44,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:44,319 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 94f2bd42818e4b89b48d1ecea55c5c6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732121561683 2024-11-20T16:52:44,320 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting d5ad19dd112a430cb1ac3695bf2907df, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732121562335 2024-11-20T16:52:44,320 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cf44e08c23b42ab93bb51652adc8b8f, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732121563663 2024-11-20T16:52:44,320 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fd67cbf28c249d4a19291840a381364, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732121563663 2024-11-20T16:52:44,343 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:44,351 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#B#compaction#136 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:44,351 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/cd25ff59abb54b74b51eb23ae5459ebb is 50, key is test_row_0/B:col10/1732121563697/Put/seqid=0 2024-11-20T16:52:44,355 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411205c8bc1f6e9f54395802d1bcbe68ca927_94c122f231a59b575ec7f40dae2c00f4 store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:44,359 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411205c8bc1f6e9f54395802d1bcbe68ca927_94c122f231a59b575ec7f40dae2c00f4, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:44,360 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205c8bc1f6e9f54395802d1bcbe68ca927_94c122f231a59b575ec7f40dae2c00f4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:44,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741980_1156 (size=12223) 2024-11-20T16:52:44,393 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/cd25ff59abb54b74b51eb23ae5459ebb as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/cd25ff59abb54b74b51eb23ae5459ebb 2024-11-20T16:52:44,405 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/B of 94c122f231a59b575ec7f40dae2c00f4 into cd25ff59abb54b74b51eb23ae5459ebb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:44,405 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:44,406 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/B, priority=11, startTime=1732121564312; duration=0sec 2024-11-20T16:52:44,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741981_1157 (size=4469) 2024-11-20T16:52:44,409 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:44,409 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:B 2024-11-20T16:52:44,409 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T16:52:44,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T16:52:44,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:44,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:44,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:44,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:44,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:44,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:44,411 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57711 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T16:52:44,411 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/C is initiating minor compaction (all files) 2024-11-20T16:52:44,411 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/C in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:44,411 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/cd2cc91b51e247ebbf2b725957e2eb7c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/dda406cb62594dff9d41e7b4dffbe34d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/8e737d7a89ed467d86382c1af0801e0a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/3058e11aa4b4441d822d46e501fc538e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/aa834389a15d4822bf6fbaa2bc93d800] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=56.4 K 2024-11-20T16:52:44,412 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting cd2cc91b51e247ebbf2b725957e2eb7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732121561193 2024-11-20T16:52:44,412 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting dda406cb62594dff9d41e7b4dffbe34d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732121561262 2024-11-20T16:52:44,412 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e737d7a89ed467d86382c1af0801e0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732121561683 2024-11-20T16:52:44,413 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 3058e11aa4b4441d822d46e501fc538e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732121562335 2024-11-20T16:52:44,413 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting aa834389a15d4822bf6fbaa2bc93d800, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732121563663 2024-11-20T16:52:44,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T16:52:44,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:44,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:44,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:44,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120be5f9f2a916a490db1a7cbf21aca0c49_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121563754/Put/seqid=0 2024-11-20T16:52:44,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:44,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:44,442 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#C#compaction#138 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:44,443 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/6df5e8b2a8964392a2678e129016f77e is 50, key is test_row_0/C:col10/1732121563697/Put/seqid=0 2024-11-20T16:52:44,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741983_1159 (size=12223) 2024-11-20T16:52:44,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121624447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:44,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:44,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741982_1158 (size=14794) 2024-11-20T16:52:44,461 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,462 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/6df5e8b2a8964392a2678e129016f77e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/6df5e8b2a8964392a2678e129016f77e 2024-11-20T16:52:44,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121624457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121624458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:44,469 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120be5f9f2a916a490db1a7cbf21aca0c49_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120be5f9f2a916a490db1a7cbf21aca0c49_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:44,470 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/b1034b44bce24503b6c10b063a49e75c, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:44,470 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/C of 94c122f231a59b575ec7f40dae2c00f4 into 6df5e8b2a8964392a2678e129016f77e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:44,470 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:44,470 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/C, priority=11, startTime=1732121564318; duration=0sec 2024-11-20T16:52:44,470 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:44,470 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:C 2024-11-20T16:52:44,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/b1034b44bce24503b6c10b063a49e75c is 175, key is test_row_0/A:col10/1732121563754/Put/seqid=0 2024-11-20T16:52:44,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741984_1160 (size=39749) 2024-11-20T16:52:44,521 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=150, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/b1034b44bce24503b6c10b063a49e75c 2024-11-20T16:52:44,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/f7369e6561ef45cf90fbdc773d90cbb1 is 50, key is test_row_0/B:col10/1732121563754/Put/seqid=0 2024-11-20T16:52:44,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741985_1161 (size=12151) 2024-11-20T16:52:44,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/f7369e6561ef45cf90fbdc773d90cbb1 2024-11-20T16:52:44,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/d30aa1da9d4b4163b96770292fb7ffc4 is 50, key is test_row_0/C:col10/1732121563754/Put/seqid=0 2024-11-20T16:52:44,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121624566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:44,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:44,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121624576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121624576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:44,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T16:52:44,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:44,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:44,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:44,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:44,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T16:52:44,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741986_1162 (size=12151) 2024-11-20T16:52:44,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/d30aa1da9d4b4163b96770292fb7ffc4 2024-11-20T16:52:44,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/b1034b44bce24503b6c10b063a49e75c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b1034b44bce24503b6c10b063a49e75c 2024-11-20T16:52:44,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b1034b44bce24503b6c10b063a49e75c, entries=200, sequenceid=150, filesize=38.8 K 2024-11-20T16:52:44,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/f7369e6561ef45cf90fbdc773d90cbb1 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/f7369e6561ef45cf90fbdc773d90cbb1 2024-11-20T16:52:44,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/f7369e6561ef45cf90fbdc773d90cbb1, entries=150, sequenceid=150, filesize=11.9 K 2024-11-20T16:52:44,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/d30aa1da9d4b4163b96770292fb7ffc4 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/d30aa1da9d4b4163b96770292fb7ffc4 2024-11-20T16:52:44,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/d30aa1da9d4b4163b96770292fb7ffc4, entries=150, sequenceid=150, filesize=11.9 K 2024-11-20T16:52:44,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 94c122f231a59b575ec7f40dae2c00f4 in 229ms, sequenceid=150, compaction requested=false 2024-11-20T16:52:44,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T16:52:44,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:44,736 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T16:52:44,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:44,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:44,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:44,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:44,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:44,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:44,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206c71c919d46a4ef3bdc7b0e34fe9466e_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121564454/Put/seqid=0 2024-11-20T16:52:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741987_1163 (size=9814) 2024-11-20T16:52:44,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:44,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,811 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#A#compaction#135 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:44,811 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/0b9fe730c513465ea746d13fe026c891 is 175, key is test_row_0/A:col10/1732121563697/Put/seqid=0 2024-11-20T16:52:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741988_1164 (size=31177) 2024-11-20T16:52:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:44,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121624822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:44,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:44,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121624824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:44,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:44,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,832 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/0b9fe730c513465ea746d13fe026c891 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0b9fe730c513465ea746d13fe026c891 2024-11-20T16:52:44,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,839 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/A of 94c122f231a59b575ec7f40dae2c00f4 into 0b9fe730c513465ea746d13fe026c891(size=30.4 K), total size for store is 69.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:44,839 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:44,839 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/A, priority=11, startTime=1732121564311; duration=0sec 2024-11-20T16:52:44,839 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:44,839 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:A 2024-11-20T16:52:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121624832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:44,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:44,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121624936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:44,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:44,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121624936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:44,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:44,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121624955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:44,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:44,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T16:52:45,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121625146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:45,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121625146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:45,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:45,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:45,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:45,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121625166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:45,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:45,189 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206c71c919d46a4ef3bdc7b0e34fe9466e_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206c71c919d46a4ef3bdc7b0e34fe9466e_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:45,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/ea0e27a4b39d45fa9f5da03cddb2ecd4, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:45,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/ea0e27a4b39d45fa9f5da03cddb2ecd4 is 175, key is test_row_0/A:col10/1732121564454/Put/seqid=0 2024-11-20T16:52:45,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741989_1165 (size=22461) 2024-11-20T16:52:45,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121625456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:45,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121625456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:45,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121625476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:45,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:45,625 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/ea0e27a4b39d45fa9f5da03cddb2ecd4 2024-11-20T16:52:45,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/e9a3e467f30744a4a1e0a1a665ebb94f is 50, key is test_row_0/B:col10/1732121564454/Put/seqid=0 2024-11-20T16:52:45,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741990_1166 (size=9757) 2024-11-20T16:52:45,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45236 deadline: 1732121625896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:45,896 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 94c122f231a59b575ec7f40dae2c00f4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:52:45,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:45,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45304 deadline: 1732121625896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:45,898 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 94c122f231a59b575ec7f40dae2c00f4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:52:45,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:45,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121625975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121625976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:45,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:45,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121625986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:46,036 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/e9a3e467f30744a4a1e0a1a665ebb94f 2024-11-20T16:52:46,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/dfebf2bcf86d40c9a600657244d63e27 is 50, key is test_row_0/C:col10/1732121564454/Put/seqid=0 2024-11-20T16:52:46,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741991_1167 (size=9757) 2024-11-20T16:52:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T16:52:46,448 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/dfebf2bcf86d40c9a600657244d63e27 2024-11-20T16:52:46,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/ea0e27a4b39d45fa9f5da03cddb2ecd4 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/ea0e27a4b39d45fa9f5da03cddb2ecd4 2024-11-20T16:52:46,456 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/ea0e27a4b39d45fa9f5da03cddb2ecd4, entries=100, sequenceid=166, filesize=21.9 K 2024-11-20T16:52:46,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/e9a3e467f30744a4a1e0a1a665ebb94f as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/e9a3e467f30744a4a1e0a1a665ebb94f 2024-11-20T16:52:46,462 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/e9a3e467f30744a4a1e0a1a665ebb94f, entries=100, sequenceid=166, filesize=9.5 K 2024-11-20T16:52:46,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/dfebf2bcf86d40c9a600657244d63e27 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/dfebf2bcf86d40c9a600657244d63e27 2024-11-20T16:52:46,471 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/dfebf2bcf86d40c9a600657244d63e27, entries=100, sequenceid=166, filesize=9.5 K 2024-11-20T16:52:46,481 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 94c122f231a59b575ec7f40dae2c00f4 in 1744ms, sequenceid=166, compaction requested=true 2024-11-20T16:52:46,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:46,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:46,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-20T16:52:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-20T16:52:46,489 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-20T16:52:46,489 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5250 sec 2024-11-20T16:52:46,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.5330 sec 2024-11-20T16:52:46,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:46,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T16:52:46,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:46,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:46,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:46,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:46,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:46,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:47,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204be369d75ce04b5688135d24eb224ef4_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121564820/Put/seqid=0 2024-11-20T16:52:47,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741992_1168 (size=14794) 2024-11-20T16:52:47,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121627055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:47,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:47,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121627077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:47,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121627083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:47,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:47,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121627177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:47,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:47,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:47,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121627196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:47,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:47,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:47,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121627196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:47,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121627396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:47,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121627417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:47,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:47,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121627417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:47,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:47,461 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:47,469 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204be369d75ce04b5688135d24eb224ef4_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204be369d75ce04b5688135d24eb224ef4_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:47,470 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/c14edccfe7bd4e2592f6c5c3fc7d5478, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:47,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/c14edccfe7bd4e2592f6c5c3fc7d5478 is 175, key is test_row_0/A:col10/1732121564820/Put/seqid=0 2024-11-20T16:52:47,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741993_1169 (size=39749) 2024-11-20T16:52:47,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121627716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:47,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:47,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121627726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:47,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:47,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121627736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:47,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:47,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:47,914 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=185, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/c14edccfe7bd4e2592f6c5c3fc7d5478 2024-11-20T16:52:47,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/4f1da40bdfb241328ee3e073fef3991f is 50, key is test_row_0/B:col10/1732121564820/Put/seqid=0 2024-11-20T16:52:48,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741994_1170 (size=12151) 2024-11-20T16:52:48,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/4f1da40bdfb241328ee3e073fef3991f 2024-11-20T16:52:48,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/56a7c40ae8ac4ef0a721852a946ba1e9 is 50, key is test_row_0/C:col10/1732121564820/Put/seqid=0 2024-11-20T16:52:48,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741995_1171 (size=12151) 2024-11-20T16:52:48,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/56a7c40ae8ac4ef0a721852a946ba1e9 2024-11-20T16:52:48,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/c14edccfe7bd4e2592f6c5c3fc7d5478 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/c14edccfe7bd4e2592f6c5c3fc7d5478 2024-11-20T16:52:48,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/c14edccfe7bd4e2592f6c5c3fc7d5478, entries=200, sequenceid=185, filesize=38.8 K 2024-11-20T16:52:48,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/4f1da40bdfb241328ee3e073fef3991f as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/4f1da40bdfb241328ee3e073fef3991f 2024-11-20T16:52:48,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/4f1da40bdfb241328ee3e073fef3991f, entries=150, sequenceid=185, filesize=11.9 K 2024-11-20T16:52:48,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/56a7c40ae8ac4ef0a721852a946ba1e9 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/56a7c40ae8ac4ef0a721852a946ba1e9 2024-11-20T16:52:48,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/56a7c40ae8ac4ef0a721852a946ba1e9, entries=150, sequenceid=185, filesize=11.9 K 2024-11-20T16:52:48,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 94c122f231a59b575ec7f40dae2c00f4 in 1133ms, sequenceid=185, compaction requested=true 2024-11-20T16:52:48,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:48,124 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:52:48,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:48,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:48,124 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:52:48,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:48,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:48,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:48,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T16:52:48,126 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:48,127 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133136 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:52:48,127 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/A is initiating minor compaction (all files) 2024-11-20T16:52:48,127 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/A in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:48,127 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0b9fe730c513465ea746d13fe026c891, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b1034b44bce24503b6c10b063a49e75c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/ea0e27a4b39d45fa9f5da03cddb2ecd4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/c14edccfe7bd4e2592f6c5c3fc7d5478] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=130.0 K 2024-11-20T16:52:48,128 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:48,128 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0b9fe730c513465ea746d13fe026c891, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b1034b44bce24503b6c10b063a49e75c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/ea0e27a4b39d45fa9f5da03cddb2ecd4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/c14edccfe7bd4e2592f6c5c3fc7d5478] 2024-11-20T16:52:48,128 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:48,129 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46282 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:52:48,129 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/B is initiating minor compaction (all files) 2024-11-20T16:52:48,129 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/B in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:48,130 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/cd25ff59abb54b74b51eb23ae5459ebb, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/f7369e6561ef45cf90fbdc773d90cbb1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/e9a3e467f30744a4a1e0a1a665ebb94f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/4f1da40bdfb241328ee3e073fef3991f] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=45.2 K 2024-11-20T16:52:48,131 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b9fe730c513465ea746d13fe026c891, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732121563663 2024-11-20T16:52:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-20T16:52:48,133 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T16:52:48,135 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:48,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:48,135 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting cd25ff59abb54b74b51eb23ae5459ebb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732121563663 2024-11-20T16:52:48,136 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1034b44bce24503b6c10b063a49e75c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732121563752 2024-11-20T16:52:48,136 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting f7369e6561ef45cf90fbdc773d90cbb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732121563753 2024-11-20T16:52:48,137 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting e9a3e467f30744a4a1e0a1a665ebb94f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732121564445 2024-11-20T16:52:48,137 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea0e27a4b39d45fa9f5da03cddb2ecd4, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732121564445 2024-11-20T16:52:48,137 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting c14edccfe7bd4e2592f6c5c3fc7d5478, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732121564818 2024-11-20T16:52:48,138 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f1da40bdfb241328ee3e073fef3991f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732121564820 2024-11-20T16:52:48,167 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#B#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:48,168 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/a0d629637c4343d3a17583e53397cfd3 is 50, key is test_row_0/B:col10/1732121564820/Put/seqid=0 2024-11-20T16:52:48,187 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:48,215 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120d3bebf98c0c44aacae8884786e66fcc5_94c122f231a59b575ec7f40dae2c00f4 store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:48,225 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120d3bebf98c0c44aacae8884786e66fcc5_94c122f231a59b575ec7f40dae2c00f4, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:48,225 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d3bebf98c0c44aacae8884786e66fcc5_94c122f231a59b575ec7f40dae2c00f4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T16:52:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:48,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T16:52:48,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:48,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:48,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:48,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:48,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:48,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:48,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741996_1172 (size=12459) 2024-11-20T16:52:48,261 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/a0d629637c4343d3a17583e53397cfd3 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/a0d629637c4343d3a17583e53397cfd3 2024-11-20T16:52:48,269 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/B of 94c122f231a59b575ec7f40dae2c00f4 into a0d629637c4343d3a17583e53397cfd3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:48,269 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:48,269 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/B, priority=12, startTime=1732121568124; duration=0sec 2024-11-20T16:52:48,269 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:48,269 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:B 2024-11-20T16:52:48,269 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:52:48,270 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46282 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:52:48,270 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/C is initiating minor compaction (all files) 2024-11-20T16:52:48,271 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/C in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:48,271 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/6df5e8b2a8964392a2678e129016f77e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/d30aa1da9d4b4163b96770292fb7ffc4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/dfebf2bcf86d40c9a600657244d63e27, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/56a7c40ae8ac4ef0a721852a946ba1e9] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=45.2 K 2024-11-20T16:52:48,272 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 6df5e8b2a8964392a2678e129016f77e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732121563663 2024-11-20T16:52:48,273 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting d30aa1da9d4b4163b96770292fb7ffc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732121563753 2024-11-20T16:52:48,274 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting dfebf2bcf86d40c9a600657244d63e27, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732121564445 2024-11-20T16:52:48,274 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 56a7c40ae8ac4ef0a721852a946ba1e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732121564820 2024-11-20T16:52:48,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a7a644824d5d440b9fcf7d21630c2674_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121567064/Put/seqid=0 2024-11-20T16:52:48,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T16:52:48,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:48,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:48,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:48,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:48,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:48,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741997_1173 (size=4469) 2024-11-20T16:52:48,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121628318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:48,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121628318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:48,329 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#C#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:48,330 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/e700478c5ce749febfcd4a126e46af0e is 50, key is test_row_0/C:col10/1732121564820/Put/seqid=0 2024-11-20T16:52:48,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:48,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121628321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:48,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:48,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741998_1174 (size=14794) 2024-11-20T16:52:48,358 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:48,365 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a7a644824d5d440b9fcf7d21630c2674_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a7a644824d5d440b9fcf7d21630c2674_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:48,368 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/bf8db4e66c094f888417ed064a3a9e94, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:48,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/bf8db4e66c094f888417ed064a3a9e94 is 175, key is test_row_0/A:col10/1732121567064/Put/seqid=0 2024-11-20T16:52:48,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741999_1175 (size=12459) 2024-11-20T16:52:48,403 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/e700478c5ce749febfcd4a126e46af0e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/e700478c5ce749febfcd4a126e46af0e 2024-11-20T16:52:48,411 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/C of 94c122f231a59b575ec7f40dae2c00f4 into e700478c5ce749febfcd4a126e46af0e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:48,411 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:48,411 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/C, priority=12, startTime=1732121568125; duration=0sec 2024-11-20T16:52:48,411 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:48,411 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:C 2024-11-20T16:52:48,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742000_1176 (size=39749) 2024-11-20T16:52:48,417 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/bf8db4e66c094f888417ed064a3a9e94 2024-11-20T16:52:48,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T16:52:48,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:48,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:48,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:48,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:48,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:48,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121628436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:48,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121628436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T16:52:48,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/6c81ffed432544f9ba36b02a15452b82 is 50, key is test_row_0/B:col10/1732121567064/Put/seqid=0 2024-11-20T16:52:48,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121628446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:48,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742001_1177 (size=12151) 2024-11-20T16:52:48,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/6c81ffed432544f9ba36b02a15452b82 2024-11-20T16:52:48,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/56fd725fcbdf43d38867f9f9d9994db8 is 50, key is test_row_0/C:col10/1732121567064/Put/seqid=0 2024-11-20T16:52:48,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742002_1178 (size=12151) 2024-11-20T16:52:48,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/56fd725fcbdf43d38867f9f9d9994db8 2024-11-20T16:52:48,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/bf8db4e66c094f888417ed064a3a9e94 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/bf8db4e66c094f888417ed064a3a9e94 2024-11-20T16:52:48,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/bf8db4e66c094f888417ed064a3a9e94, entries=200, sequenceid=201, filesize=38.8 K 2024-11-20T16:52:48,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/6c81ffed432544f9ba36b02a15452b82 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/6c81ffed432544f9ba36b02a15452b82 2024-11-20T16:52:48,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/6c81ffed432544f9ba36b02a15452b82, entries=150, sequenceid=201, filesize=11.9 K 2024-11-20T16:52:48,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/56fd725fcbdf43d38867f9f9d9994db8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/56fd725fcbdf43d38867f9f9d9994db8 2024-11-20T16:52:48,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/56fd725fcbdf43d38867f9f9d9994db8, entries=150, sequenceid=201, filesize=11.9 K 2024-11-20T16:52:48,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 94c122f231a59b575ec7f40dae2c00f4 in 331ms, sequenceid=201, compaction requested=false 2024-11-20T16:52:48,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:48,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T16:52:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:48,598 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T16:52:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:48,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:48,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:48,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a01e5e5ba3cf4d53b68e506139eef1b5_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121568294/Put/seqid=0 2024-11-20T16:52:48,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742003_1179 (size=12304) 2024-11-20T16:52:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:48,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:48,711 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#A#compaction#148 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:48,711 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/7ce54154e9ac4f09a8b06ec37f1a536b is 175, key is test_row_0/A:col10/1732121564820/Put/seqid=0 2024-11-20T16:52:48,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:48,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121628717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:48,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:48,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121628717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:48,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:48,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121628729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:48,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:48,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742004_1180 (size=31413) 2024-11-20T16:52:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T16:52:48,768 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/7ce54154e9ac4f09a8b06ec37f1a536b as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/7ce54154e9ac4f09a8b06ec37f1a536b 2024-11-20T16:52:48,773 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/A of 94c122f231a59b575ec7f40dae2c00f4 into 7ce54154e9ac4f09a8b06ec37f1a536b(size=30.7 K), total size for store is 69.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:48,773 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:48,773 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/A, priority=12, startTime=1732121568124; duration=0sec 2024-11-20T16:52:48,774 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:48,774 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:A 2024-11-20T16:52:48,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:48,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121628836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:48,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121628847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:48,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121628848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:48,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:48,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:49,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121629046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:49,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:49,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121629057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:49,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121629057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:49,073 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a01e5e5ba3cf4d53b68e506139eef1b5_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a01e5e5ba3cf4d53b68e506139eef1b5_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:49,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/a4cae39517864a528fc714ca54690ccd, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:49,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/a4cae39517864a528fc714ca54690ccd is 175, key is test_row_0/A:col10/1732121568294/Put/seqid=0 2024-11-20T16:52:49,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742005_1181 (size=31105) 2024-11-20T16:52:49,104 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/a4cae39517864a528fc714ca54690ccd 2024-11-20T16:52:49,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/00f226bc322d446ab6c56aae80c1a63f is 50, key is test_row_0/B:col10/1732121568294/Put/seqid=0 2024-11-20T16:52:49,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742006_1182 (size=12151) 2024-11-20T16:52:49,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T16:52:49,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:49,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121629356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:49,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:49,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121629366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:49,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121629366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:49,584 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/00f226bc322d446ab6c56aae80c1a63f 2024-11-20T16:52:49,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/d46f5a97021a481cab6a1dcebb5b197a is 50, key is test_row_0/C:col10/1732121568294/Put/seqid=0 2024-11-20T16:52:49,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742007_1183 (size=12151) 2024-11-20T16:52:49,600 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/d46f5a97021a481cab6a1dcebb5b197a 2024-11-20T16:52:49,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/a4cae39517864a528fc714ca54690ccd as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a4cae39517864a528fc714ca54690ccd 2024-11-20T16:52:49,610 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a4cae39517864a528fc714ca54690ccd, entries=150, sequenceid=219, filesize=30.4 K 2024-11-20T16:52:49,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/00f226bc322d446ab6c56aae80c1a63f as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/00f226bc322d446ab6c56aae80c1a63f 2024-11-20T16:52:49,617 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/00f226bc322d446ab6c56aae80c1a63f, entries=150, sequenceid=219, filesize=11.9 K 2024-11-20T16:52:49,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/d46f5a97021a481cab6a1dcebb5b197a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/d46f5a97021a481cab6a1dcebb5b197a 2024-11-20T16:52:49,631 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/d46f5a97021a481cab6a1dcebb5b197a, entries=150, sequenceid=219, filesize=11.9 K 2024-11-20T16:52:49,632 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 94c122f231a59b575ec7f40dae2c00f4 in 1034ms, sequenceid=219, compaction requested=true 2024-11-20T16:52:49,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:49,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:49,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-20T16:52:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-20T16:52:49,635 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-20T16:52:49,635 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4990 sec 2024-11-20T16:52:49,641 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.5090 sec 2024-11-20T16:52:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:49,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T16:52:49,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:49,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:49,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:49,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:49,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:49,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:49,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e7e77fe4c94f46f7830ab8464f9d9ea0_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121569869/Put/seqid=0 2024-11-20T16:52:49,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742008_1184 (size=14794) 2024-11-20T16:52:49,885 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:49,889 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e7e77fe4c94f46f7830ab8464f9d9ea0_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e7e77fe4c94f46f7830ab8464f9d9ea0_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:49,890 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/a7ed4166bb714da3adcd077b5a522a6a, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:49,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/a7ed4166bb714da3adcd077b5a522a6a is 175, key is test_row_0/A:col10/1732121569869/Put/seqid=0 2024-11-20T16:52:49,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742009_1185 (size=39749) 2024-11-20T16:52:49,898 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/a7ed4166bb714da3adcd077b5a522a6a 2024-11-20T16:52:49,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121629899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:49,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:49,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121629900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:49,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:49,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/9ca2ae43eec4416cb37d32e7994b416c is 50, key is test_row_0/B:col10/1732121569869/Put/seqid=0 2024-11-20T16:52:49,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121629900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:49,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742010_1186 (size=12151) 2024-11-20T16:52:49,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45236 deadline: 1732121629907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:49,914 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 94c122f231a59b575ec7f40dae2c00f4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:52:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:49,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45304 deadline: 1732121629916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:49,918 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 94c122f231a59b575ec7f40dae2c00f4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:52:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:50,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121630007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:50,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:50,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:50,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121630018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121630021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:50,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:50,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121630216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:50,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:50,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121630226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:50,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121630226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:50,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:50,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:50,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T16:52:50,277 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:50,277 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:50,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-20T16:52:50,279 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:50,281 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:50,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:50,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T16:52:50,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/9ca2ae43eec4416cb37d32e7994b416c 2024-11-20T16:52:50,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/6d16fa124b044b39a04301d622e1e8de is 50, key is test_row_0/C:col10/1732121569869/Put/seqid=0 2024-11-20T16:52:50,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742011_1187 (size=12151) 2024-11-20T16:52:50,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/6d16fa124b044b39a04301d622e1e8de 2024-11-20T16:52:50,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/a7ed4166bb714da3adcd077b5a522a6a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a7ed4166bb714da3adcd077b5a522a6a 2024-11-20T16:52:50,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a7ed4166bb714da3adcd077b5a522a6a, entries=200, sequenceid=237, filesize=38.8 K 2024-11-20T16:52:50,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/9ca2ae43eec4416cb37d32e7994b416c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/9ca2ae43eec4416cb37d32e7994b416c 2024-11-20T16:52:50,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T16:52:50,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/9ca2ae43eec4416cb37d32e7994b416c, entries=150, sequenceid=237, filesize=11.9 K 2024-11-20T16:52:50,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/6d16fa124b044b39a04301d622e1e8de as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/6d16fa124b044b39a04301d622e1e8de 2024-11-20T16:52:50,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/6d16fa124b044b39a04301d622e1e8de, entries=150, sequenceid=237, filesize=11.9 K 2024-11-20T16:52:50,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 94c122f231a59b575ec7f40dae2c00f4 in 539ms, sequenceid=237, compaction requested=true 2024-11-20T16:52:50,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:50,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:50,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:50,409 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:52:50,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:50,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:50,410 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:52:50,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:50,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:50,411 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:52:50,411 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/A is initiating minor compaction (all files) 2024-11-20T16:52:50,411 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:52:50,411 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/B is initiating minor compaction (all files) 2024-11-20T16:52:50,411 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/A in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:50,411 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/B in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:50,411 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/7ce54154e9ac4f09a8b06ec37f1a536b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/bf8db4e66c094f888417ed064a3a9e94, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a4cae39517864a528fc714ca54690ccd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a7ed4166bb714da3adcd077b5a522a6a] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=138.7 K 2024-11-20T16:52:50,411 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/a0d629637c4343d3a17583e53397cfd3, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/6c81ffed432544f9ba36b02a15452b82, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/00f226bc322d446ab6c56aae80c1a63f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/9ca2ae43eec4416cb37d32e7994b416c] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=47.8 K 2024-11-20T16:52:50,411 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:50,412 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/7ce54154e9ac4f09a8b06ec37f1a536b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/bf8db4e66c094f888417ed064a3a9e94, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a4cae39517864a528fc714ca54690ccd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a7ed4166bb714da3adcd077b5a522a6a] 2024-11-20T16:52:50,412 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting a0d629637c4343d3a17583e53397cfd3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732121564820 2024-11-20T16:52:50,412 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ce54154e9ac4f09a8b06ec37f1a536b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732121564820 2024-11-20T16:52:50,412 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c81ffed432544f9ba36b02a15452b82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732121567047 2024-11-20T16:52:50,413 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf8db4e66c094f888417ed064a3a9e94, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732121567023 2024-11-20T16:52:50,413 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 00f226bc322d446ab6c56aae80c1a63f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732121568280 2024-11-20T16:52:50,413 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4cae39517864a528fc714ca54690ccd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732121568280 2024-11-20T16:52:50,413 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ca2ae43eec4416cb37d32e7994b416c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732121568707 2024-11-20T16:52:50,413 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7ed4166bb714da3adcd077b5a522a6a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732121568707 2024-11-20T16:52:50,430 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#B#compaction#159 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:50,430 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/a95616ac925d4c54b5351153ba78bec0 is 50, key is test_row_0/B:col10/1732121569869/Put/seqid=0 2024-11-20T16:52:50,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T16:52:50,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:50,436 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T16:52:50,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:50,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:50,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:50,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:50,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:50,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:50,439 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:50,458 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120998c194afc3d4202a045d568841b4f9b_94c122f231a59b575ec7f40dae2c00f4 store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:50,462 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120998c194afc3d4202a045d568841b4f9b_94c122f231a59b575ec7f40dae2c00f4, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:50,462 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120998c194afc3d4202a045d568841b4f9b_94c122f231a59b575ec7f40dae2c00f4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:50,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112046351b3fc3d2475488ca304aee5eeff1_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121569893/Put/seqid=0 2024-11-20T16:52:50,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742012_1188 (size=12595) 2024-11-20T16:52:50,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742013_1189 (size=4469) 2024-11-20T16:52:50,491 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#A#compaction#160 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:50,491 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/b250a3a08ab6423fa16d95c4e88a5da7 is 175, key is test_row_0/A:col10/1732121569869/Put/seqid=0 2024-11-20T16:52:50,494 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/a95616ac925d4c54b5351153ba78bec0 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/a95616ac925d4c54b5351153ba78bec0 2024-11-20T16:52:50,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742014_1190 (size=12304) 2024-11-20T16:52:50,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:50,501 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/B of 94c122f231a59b575ec7f40dae2c00f4 into a95616ac925d4c54b5351153ba78bec0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:50,501 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:50,501 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/B, priority=12, startTime=1732121570409; duration=0sec 2024-11-20T16:52:50,501 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:50,501 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:B 2024-11-20T16:52:50,501 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:52:50,504 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:52:50,504 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/C is initiating minor compaction (all files) 2024-11-20T16:52:50,504 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/C in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:50,504 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/e700478c5ce749febfcd4a126e46af0e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/56fd725fcbdf43d38867f9f9d9994db8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/d46f5a97021a481cab6a1dcebb5b197a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/6d16fa124b044b39a04301d622e1e8de] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=47.8 K 2024-11-20T16:52:50,505 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting e700478c5ce749febfcd4a126e46af0e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732121564820 2024-11-20T16:52:50,505 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 56fd725fcbdf43d38867f9f9d9994db8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732121567047 2024-11-20T16:52:50,506 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting d46f5a97021a481cab6a1dcebb5b197a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732121568280 2024-11-20T16:52:50,506 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112046351b3fc3d2475488ca304aee5eeff1_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112046351b3fc3d2475488ca304aee5eeff1_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:50,507 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d16fa124b044b39a04301d622e1e8de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732121568707 2024-11-20T16:52:50,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/0da8b82ab450495288e560ed059a5b24, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:50,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/0da8b82ab450495288e560ed059a5b24 is 175, key is test_row_0/A:col10/1732121569893/Put/seqid=0 2024-11-20T16:52:50,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742015_1191 (size=31549) 2024-11-20T16:52:50,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742016_1192 (size=31105) 2024-11-20T16:52:50,522 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/0da8b82ab450495288e560ed059a5b24 2024-11-20T16:52:50,530 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#C#compaction#162 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:50,531 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/9f2430e4fb2e484395c42af887648055 is 50, key is test_row_0/C:col10/1732121569869/Put/seqid=0 2024-11-20T16:52:50,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/2f4c5a13c206403e90e1457975a3f24c is 50, key is test_row_0/B:col10/1732121569893/Put/seqid=0 2024-11-20T16:52:50,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742017_1193 (size=12595) 2024-11-20T16:52:50,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:50,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:50,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742018_1194 (size=12151) 2024-11-20T16:52:50,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:50,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121630587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:50,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:50,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121630588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:50,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T16:52:50,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121630593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:50,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121630696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:50,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:50,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121630706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121630706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T16:52:50,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121630906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:50,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121630916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:50,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121630916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:50,932 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/b250a3a08ab6423fa16d95c4e88a5da7 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b250a3a08ab6423fa16d95c4e88a5da7 2024-11-20T16:52:50,941 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/A of 94c122f231a59b575ec7f40dae2c00f4 into b250a3a08ab6423fa16d95c4e88a5da7(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:50,941 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:50,941 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/A, priority=12, startTime=1732121570409; duration=0sec 2024-11-20T16:52:50,942 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:50,942 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:A 2024-11-20T16:52:50,957 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/9f2430e4fb2e484395c42af887648055 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/9f2430e4fb2e484395c42af887648055 2024-11-20T16:52:50,965 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/C of 94c122f231a59b575ec7f40dae2c00f4 into 9f2430e4fb2e484395c42af887648055(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:50,965 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:50,965 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/C, priority=12, startTime=1732121570410; duration=0sec 2024-11-20T16:52:50,965 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:50,966 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:C 2024-11-20T16:52:50,966 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/2f4c5a13c206403e90e1457975a3f24c 2024-11-20T16:52:50,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/609d9a1e52ae48629924d0728c393359 is 50, key is test_row_0/C:col10/1732121569893/Put/seqid=0 2024-11-20T16:52:50,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742019_1195 (size=12151) 2024-11-20T16:52:50,998 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/609d9a1e52ae48629924d0728c393359 2024-11-20T16:52:51,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/0da8b82ab450495288e560ed059a5b24 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0da8b82ab450495288e560ed059a5b24 2024-11-20T16:52:51,012 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0da8b82ab450495288e560ed059a5b24, entries=150, sequenceid=252, filesize=30.4 K 2024-11-20T16:52:51,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/2f4c5a13c206403e90e1457975a3f24c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/2f4c5a13c206403e90e1457975a3f24c 2024-11-20T16:52:51,026 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/2f4c5a13c206403e90e1457975a3f24c, entries=150, sequenceid=252, filesize=11.9 K 2024-11-20T16:52:51,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/609d9a1e52ae48629924d0728c393359 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/609d9a1e52ae48629924d0728c393359 2024-11-20T16:52:51,031 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/609d9a1e52ae48629924d0728c393359, entries=150, sequenceid=252, filesize=11.9 K 2024-11-20T16:52:51,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-20T16:52:51,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-20T16:52:51,035 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 94c122f231a59b575ec7f40dae2c00f4 in 599ms, sequenceid=252, compaction requested=false 2024-11-20T16:52:51,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:51,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:51,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-20T16:52:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-20T16:52:51,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-20T16:52:51,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 755 msec 2024-11-20T16:52:51,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 761 msec 2024-11-20T16:52:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:51,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T16:52:51,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:51,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:51,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:51,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:51,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:51,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:51,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d1ba5828cea244f4a15be52831dd59c8_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121570591/Put/seqid=0 2024-11-20T16:52:51,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742020_1196 (size=14994) 2024-11-20T16:52:51,257 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:51,262 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d1ba5828cea244f4a15be52831dd59c8_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d1ba5828cea244f4a15be52831dd59c8_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:51,263 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/5148a7260afd4f3da30af261eda52723, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:51,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/5148a7260afd4f3da30af261eda52723 is 175, key is test_row_0/A:col10/1732121570591/Put/seqid=0 2024-11-20T16:52:51,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742021_1197 (size=39949) 2024-11-20T16:52:51,272 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/5148a7260afd4f3da30af261eda52723 2024-11-20T16:52:51,274 INFO [master/b60eac9bda43:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-20T16:52:51,274 INFO [master/b60eac9bda43:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-20T16:52:51,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/db74a94fb9ff41fda19afafc30f2ae06 is 50, key is test_row_0/B:col10/1732121570591/Put/seqid=0 2024-11-20T16:52:51,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121631291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:51,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121631291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:51,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121631299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:51,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742022_1198 (size=12301) 2024-11-20T16:52:51,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121631406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:51,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121631406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:51,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:51,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T16:52:51,417 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:51,417 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-20T16:52:51,419 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:51,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121631417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T16:52:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:51,421 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:51,421 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T16:52:51,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T16:52:51,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:51,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:51,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:51,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:51,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:51,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121631616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:51,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121631616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:51,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:51,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121631626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:51,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T16:52:51,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:51,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:51,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:51,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:51,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T16:52:51,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/db74a94fb9ff41fda19afafc30f2ae06 2024-11-20T16:52:51,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/2d607505a10142eebcbb2710faa7cf6d is 50, key is test_row_0/C:col10/1732121570591/Put/seqid=0 2024-11-20T16:52:51,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742023_1199 (size=12301) 2024-11-20T16:52:51,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/2d607505a10142eebcbb2710faa7cf6d 2024-11-20T16:52:51,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/5148a7260afd4f3da30af261eda52723 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/5148a7260afd4f3da30af261eda52723 2024-11-20T16:52:51,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/5148a7260afd4f3da30af261eda52723, entries=200, sequenceid=273, filesize=39.0 K 2024-11-20T16:52:51,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/db74a94fb9ff41fda19afafc30f2ae06 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/db74a94fb9ff41fda19afafc30f2ae06 2024-11-20T16:52:51,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/db74a94fb9ff41fda19afafc30f2ae06, entries=150, sequenceid=273, filesize=12.0 K 2024-11-20T16:52:51,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/2d607505a10142eebcbb2710faa7cf6d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/2d607505a10142eebcbb2710faa7cf6d 2024-11-20T16:52:51,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/2d607505a10142eebcbb2710faa7cf6d, entries=150, sequenceid=273, filesize=12.0 K 2024-11-20T16:52:51,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 94c122f231a59b575ec7f40dae2c00f4 in 613ms, sequenceid=273, compaction requested=true 2024-11-20T16:52:51,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:51,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:51,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:51,836 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:51,836 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:51,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:51,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:51,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:51,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:51,837 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:51,837 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102603 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:51,837 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/A is initiating minor compaction (all files) 2024-11-20T16:52:51,837 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/B is initiating minor compaction (all files) 2024-11-20T16:52:51,838 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/B in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:51,838 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/A in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:51,838 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/a95616ac925d4c54b5351153ba78bec0, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/2f4c5a13c206403e90e1457975a3f24c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/db74a94fb9ff41fda19afafc30f2ae06] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=36.2 K 2024-11-20T16:52:51,838 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b250a3a08ab6423fa16d95c4e88a5da7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0da8b82ab450495288e560ed059a5b24, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/5148a7260afd4f3da30af261eda52723] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=100.2 K 2024-11-20T16:52:51,838 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:51,838 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b250a3a08ab6423fa16d95c4e88a5da7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0da8b82ab450495288e560ed059a5b24, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/5148a7260afd4f3da30af261eda52723] 2024-11-20T16:52:51,838 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting a95616ac925d4c54b5351153ba78bec0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732121568707 2024-11-20T16:52:51,838 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting b250a3a08ab6423fa16d95c4e88a5da7, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732121568707 2024-11-20T16:52:51,838 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f4c5a13c206403e90e1457975a3f24c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732121569893 2024-11-20T16:52:51,839 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0da8b82ab450495288e560ed059a5b24, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732121569893 2024-11-20T16:52:51,839 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting db74a94fb9ff41fda19afafc30f2ae06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732121570586 2024-11-20T16:52:51,839 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5148a7260afd4f3da30af261eda52723, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732121570585 2024-11-20T16:52:51,850 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:51,868 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#B#compaction#169 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:51,869 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/8592cb3959a4430bbe0cac2bb5da331f is 50, key is test_row_0/B:col10/1732121570591/Put/seqid=0 2024-11-20T16:52:51,871 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112092ad856eda8b44648b6cc16190a1e7e4_94c122f231a59b575ec7f40dae2c00f4 store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:51,873 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112092ad856eda8b44648b6cc16190a1e7e4_94c122f231a59b575ec7f40dae2c00f4, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:51,873 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112092ad856eda8b44648b6cc16190a1e7e4_94c122f231a59b575ec7f40dae2c00f4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:51,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T16:52:51,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:51,887 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T16:52:51,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:51,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:51,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:51,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:51,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:51,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:51,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742024_1200 (size=4469) 2024-11-20T16:52:51,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742025_1201 (size=12847) 2024-11-20T16:52:51,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206a09a8ea90814c5db194c17ae90d0412_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121571266/Put/seqid=0 2024-11-20T16:52:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:51,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:51,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742026_1202 (size=12454) 2024-11-20T16:52:51,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:51,947 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206a09a8ea90814c5db194c17ae90d0412_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206a09a8ea90814c5db194c17ae90d0412_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:51,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/f6f29ae178b64c548f50ab3f9e3e0f29, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:51,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/f6f29ae178b64c548f50ab3f9e3e0f29 is 175, key is test_row_0/A:col10/1732121571266/Put/seqid=0 2024-11-20T16:52:51,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742027_1203 (size=31255) 2024-11-20T16:52:51,979 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/f6f29ae178b64c548f50ab3f9e3e0f29 2024-11-20T16:52:51,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:51,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121631978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:51,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:51,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121631980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:51,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/b858dd0e136d4acf97e77caa24dfc799 is 50, key is test_row_0/B:col10/1732121571266/Put/seqid=0 2024-11-20T16:52:51,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:51,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121631989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:51,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:51,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:52,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742028_1204 (size=12301) 2024-11-20T16:52:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T16:52:52,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121632096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:52,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121632096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:52,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121632106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:52,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121632306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:52,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121632306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:52,314 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#A#compaction#168 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:52,314 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/16d5c59f56ab4ed2ace4dbeebf783ce2 is 175, key is test_row_0/A:col10/1732121570591/Put/seqid=0 2024-11-20T16:52:52,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121632316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:52,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742029_1205 (size=31801) 2024-11-20T16:52:52,329 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/16d5c59f56ab4ed2ace4dbeebf783ce2 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/16d5c59f56ab4ed2ace4dbeebf783ce2 2024-11-20T16:52:52,330 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/8592cb3959a4430bbe0cac2bb5da331f as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/8592cb3959a4430bbe0cac2bb5da331f 2024-11-20T16:52:52,338 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/B of 94c122f231a59b575ec7f40dae2c00f4 into 8592cb3959a4430bbe0cac2bb5da331f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:52,338 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/A of 94c122f231a59b575ec7f40dae2c00f4 into 16d5c59f56ab4ed2ace4dbeebf783ce2(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:52,338 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:52,338 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:52,338 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/B, priority=13, startTime=1732121571836; duration=0sec 2024-11-20T16:52:52,338 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/A, priority=13, startTime=1732121571836; duration=0sec 2024-11-20T16:52:52,338 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:52,338 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:B 2024-11-20T16:52:52,338 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:52,338 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:A 2024-11-20T16:52:52,338 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:52,339 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:52,339 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/C is initiating minor compaction (all files) 2024-11-20T16:52:52,339 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/C in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:52,340 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/9f2430e4fb2e484395c42af887648055, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/609d9a1e52ae48629924d0728c393359, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/2d607505a10142eebcbb2710faa7cf6d] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=36.2 K 2024-11-20T16:52:52,341 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f2430e4fb2e484395c42af887648055, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732121568707 2024-11-20T16:52:52,341 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 609d9a1e52ae48629924d0728c393359, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732121569893 2024-11-20T16:52:52,341 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d607505a10142eebcbb2710faa7cf6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732121570586 2024-11-20T16:52:52,349 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#C#compaction#172 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:52,349 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/67deb19345f2400bae46b02f944d04b3 is 50, key is test_row_0/C:col10/1732121570591/Put/seqid=0 2024-11-20T16:52:52,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742030_1206 (size=12847) 2024-11-20T16:52:52,391 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/67deb19345f2400bae46b02f944d04b3 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/67deb19345f2400bae46b02f944d04b3 2024-11-20T16:52:52,397 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/C of 94c122f231a59b575ec7f40dae2c00f4 into 67deb19345f2400bae46b02f944d04b3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:52,397 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:52,398 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/C, priority=13, startTime=1732121571836; duration=0sec 2024-11-20T16:52:52,398 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:52,398 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:C 2024-11-20T16:52:52,405 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/b858dd0e136d4acf97e77caa24dfc799 2024-11-20T16:52:52,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/dd2af85dca58427093895ef0d223393c is 50, key is test_row_0/C:col10/1732121571266/Put/seqid=0 2024-11-20T16:52:52,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742031_1207 (size=12301) 2024-11-20T16:52:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T16:52:52,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:52,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121632615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121632616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:52,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121632626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:52,858 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/dd2af85dca58427093895ef0d223393c 2024-11-20T16:52:52,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/f6f29ae178b64c548f50ab3f9e3e0f29 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f6f29ae178b64c548f50ab3f9e3e0f29 2024-11-20T16:52:52,873 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f6f29ae178b64c548f50ab3f9e3e0f29, entries=150, sequenceid=287, filesize=30.5 K 2024-11-20T16:52:52,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/b858dd0e136d4acf97e77caa24dfc799 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/b858dd0e136d4acf97e77caa24dfc799 2024-11-20T16:52:52,879 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/b858dd0e136d4acf97e77caa24dfc799, entries=150, sequenceid=287, filesize=12.0 K 2024-11-20T16:52:52,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/dd2af85dca58427093895ef0d223393c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/dd2af85dca58427093895ef0d223393c 2024-11-20T16:52:52,894 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/dd2af85dca58427093895ef0d223393c, entries=150, sequenceid=287, filesize=12.0 K 2024-11-20T16:52:52,895 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 94c122f231a59b575ec7f40dae2c00f4 in 1009ms, sequenceid=287, compaction requested=false 2024-11-20T16:52:52,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:52,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:52,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-20T16:52:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-20T16:52:52,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-20T16:52:52,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4750 sec 2024-11-20T16:52:52,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.4820 sec 2024-11-20T16:52:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:53,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T16:52:53,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:53,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:53,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:53,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:53,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:53,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:53,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204976c822146e4eb19019b454a9ac27ac_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121573129/Put/seqid=0 2024-11-20T16:52:53,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742032_1208 (size=17534) 2024-11-20T16:52:53,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121633143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:53,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:53,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121633145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:53,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:53,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:53,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121633146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:53,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:53,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:53,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121633256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:53,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:53,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:53,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121633257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:53,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:53,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:53,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121633258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:53,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121633466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:53,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121633466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:53,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:53,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121633476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:53,545 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:53,550 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204976c822146e4eb19019b454a9ac27ac_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204976c822146e4eb19019b454a9ac27ac_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:53,552 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/4bf25b1d423e4aec8fb931d7866a807c, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:53,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/4bf25b1d423e4aec8fb931d7866a807c is 175, key is test_row_0/A:col10/1732121573129/Put/seqid=0 2024-11-20T16:52:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T16:52:53,566 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:53,566 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-20T16:52:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T16:52:53,569 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:53,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742033_1209 (size=48639) 2024-11-20T16:52:53,573 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:53,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:53,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T16:52:53,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T16:52:53,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:53,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:53,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:53,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:53,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:53,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:53,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121633776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121633776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:53,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121633785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:53,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:53,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:53,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T16:52:53,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:53,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:53,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:53,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:53,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T16:52:53,974 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/4bf25b1d423e4aec8fb931d7866a807c 2024-11-20T16:52:53,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/309a76f4fe454d2fa0dc7ad57615ba1a is 50, key is test_row_0/B:col10/1732121573129/Put/seqid=0 2024-11-20T16:52:53,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742034_1210 (size=12301) 2024-11-20T16:52:53,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/309a76f4fe454d2fa0dc7ad57615ba1a 2024-11-20T16:52:54,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/bea8a3434f884f31b15a936ce6984500 is 50, key is test_row_0/C:col10/1732121573129/Put/seqid=0 2024-11-20T16:52:54,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742035_1211 (size=12301) 2024-11-20T16:52:54,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/bea8a3434f884f31b15a936ce6984500 2024-11-20T16:52:54,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/4bf25b1d423e4aec8fb931d7866a807c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/4bf25b1d423e4aec8fb931d7866a807c 2024-11-20T16:52:54,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/4bf25b1d423e4aec8fb931d7866a807c, entries=250, sequenceid=310, filesize=47.5 K 2024-11-20T16:52:54,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/309a76f4fe454d2fa0dc7ad57615ba1a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/309a76f4fe454d2fa0dc7ad57615ba1a 2024-11-20T16:52:54,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/309a76f4fe454d2fa0dc7ad57615ba1a, entries=150, sequenceid=310, filesize=12.0 K 2024-11-20T16:52:54,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/bea8a3434f884f31b15a936ce6984500 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/bea8a3434f884f31b15a936ce6984500 2024-11-20T16:52:54,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/bea8a3434f884f31b15a936ce6984500, entries=150, sequenceid=310, filesize=12.0 K 2024-11-20T16:52:54,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T16:52:54,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:54,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:54,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:54,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 94c122f231a59b575ec7f40dae2c00f4 in 904ms, sequenceid=310, compaction requested=true 2024-11-20T16:52:54,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:54,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:54,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:54,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:54,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:54,035 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:54,035 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:54,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:54,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:54,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:54,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:54,037 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111695 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:54,037 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/A is initiating minor compaction (all files) 2024-11-20T16:52:54,037 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:54,037 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/B is initiating minor compaction (all files) 2024-11-20T16:52:54,037 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/A in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:54,037 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/B in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:54,037 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/8592cb3959a4430bbe0cac2bb5da331f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/b858dd0e136d4acf97e77caa24dfc799, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/309a76f4fe454d2fa0dc7ad57615ba1a] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=36.6 K 2024-11-20T16:52:54,037 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/16d5c59f56ab4ed2ace4dbeebf783ce2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f6f29ae178b64c548f50ab3f9e3e0f29, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/4bf25b1d423e4aec8fb931d7866a807c] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=109.1 K 2024-11-20T16:52:54,037 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:54,037 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/16d5c59f56ab4ed2ace4dbeebf783ce2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f6f29ae178b64c548f50ab3f9e3e0f29, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/4bf25b1d423e4aec8fb931d7866a807c] 2024-11-20T16:52:54,037 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 8592cb3959a4430bbe0cac2bb5da331f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732121570586 2024-11-20T16:52:54,038 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16d5c59f56ab4ed2ace4dbeebf783ce2, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732121570586 2024-11-20T16:52:54,038 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting b858dd0e136d4acf97e77caa24dfc799, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732121571260 2024-11-20T16:52:54,038 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6f29ae178b64c548f50ab3f9e3e0f29, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732121571260 2024-11-20T16:52:54,038 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 309a76f4fe454d2fa0dc7ad57615ba1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732121571975 2024-11-20T16:52:54,038 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bf25b1d423e4aec8fb931d7866a807c, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732121571975 2024-11-20T16:52:54,051 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:54,058 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#B#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:54,058 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/45424478ab5045f6be398ac9d377bafa is 50, key is test_row_0/B:col10/1732121573129/Put/seqid=0 2024-11-20T16:52:54,068 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112005a13584ca864018a25d13e94781b9f5_94c122f231a59b575ec7f40dae2c00f4 store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:54,072 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112005a13584ca864018a25d13e94781b9f5_94c122f231a59b575ec7f40dae2c00f4, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:54,072 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112005a13584ca864018a25d13e94781b9f5_94c122f231a59b575ec7f40dae2c00f4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:54,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742037_1213 (size=4469) 2024-11-20T16:52:54,106 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#A#compaction#177 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:54,107 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/500e6a0a0ba841cd8eecb0999f62fad7 is 175, key is test_row_0/A:col10/1732121573129/Put/seqid=0 2024-11-20T16:52:54,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742036_1212 (size=12949) 2024-11-20T16:52:54,127 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/45424478ab5045f6be398ac9d377bafa as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/45424478ab5045f6be398ac9d377bafa 2024-11-20T16:52:54,135 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/B of 94c122f231a59b575ec7f40dae2c00f4 into 45424478ab5045f6be398ac9d377bafa(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:54,135 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:54,136 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/B, priority=13, startTime=1732121574035; duration=0sec 2024-11-20T16:52:54,136 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:54,136 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:B 2024-11-20T16:52:54,136 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:54,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742038_1214 (size=31903) 2024-11-20T16:52:54,139 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:54,139 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/C is initiating minor compaction (all files) 2024-11-20T16:52:54,139 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/C in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:54,140 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/67deb19345f2400bae46b02f944d04b3, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/dd2af85dca58427093895ef0d223393c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/bea8a3434f884f31b15a936ce6984500] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=36.6 K 2024-11-20T16:52:54,141 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 67deb19345f2400bae46b02f944d04b3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732121570586 2024-11-20T16:52:54,141 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting dd2af85dca58427093895ef0d223393c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732121571260 2024-11-20T16:52:54,144 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting bea8a3434f884f31b15a936ce6984500, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732121571975 2024-11-20T16:52:54,147 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/500e6a0a0ba841cd8eecb0999f62fad7 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/500e6a0a0ba841cd8eecb0999f62fad7 2024-11-20T16:52:54,155 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/A of 94c122f231a59b575ec7f40dae2c00f4 into 500e6a0a0ba841cd8eecb0999f62fad7(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:54,155 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:54,155 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/A, priority=13, startTime=1732121574035; duration=0sec 2024-11-20T16:52:54,155 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:54,155 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:A 2024-11-20T16:52:54,160 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#C#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:54,160 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/8ca8f765bb2945ad88d38b34d455233b is 50, key is test_row_0/C:col10/1732121573129/Put/seqid=0 2024-11-20T16:52:54,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742039_1215 (size=12949) 2024-11-20T16:52:54,188 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/8ca8f765bb2945ad88d38b34d455233b as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/8ca8f765bb2945ad88d38b34d455233b 2024-11-20T16:52:54,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T16:52:54,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:54,192 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T16:52:54,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:54,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:54,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:54,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:54,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:54,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T16:52:54,198 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/C of 94c122f231a59b575ec7f40dae2c00f4 into 8ca8f765bb2945ad88d38b34d455233b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:54,198 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:54,198 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/C, priority=13, startTime=1732121574036; duration=0sec 2024-11-20T16:52:54,198 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:54,198 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:C 2024-11-20T16:52:54,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d3e6ce7e7cd34142a40ad9ebc1373bc5_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121573144/Put/seqid=0 2024-11-20T16:52:54,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742040_1216 (size=12454) 2024-11-20T16:52:54,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:54,244 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d3e6ce7e7cd34142a40ad9ebc1373bc5_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d3e6ce7e7cd34142a40ad9ebc1373bc5_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:54,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/260649ca8cac4860968ff6e284de539e, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:54,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/260649ca8cac4860968ff6e284de539e is 175, key is test_row_0/A:col10/1732121573144/Put/seqid=0 2024-11-20T16:52:54,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742041_1217 (size=31255) 2024-11-20T16:52:54,282 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/260649ca8cac4860968ff6e284de539e 2024-11-20T16:52:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:54,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:54,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/e0e7f79d4d0447bd9a4e975ebf95efb2 is 50, key is test_row_0/B:col10/1732121573144/Put/seqid=0 2024-11-20T16:52:54,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742042_1218 (size=12301) 2024-11-20T16:52:54,317 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/e0e7f79d4d0447bd9a4e975ebf95efb2 2024-11-20T16:52:54,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/bb37cf09a16a4f15896ac50f8abd1908 is 50, key is test_row_0/C:col10/1732121573144/Put/seqid=0 2024-11-20T16:52:54,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742043_1219 (size=12301) 2024-11-20T16:52:54,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121634367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:54,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121634367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:54,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121634367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:54,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121634477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:54,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:54,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:54,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121634477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:54,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121634477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:54,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121634688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:54,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121634688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:54,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121634688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:54,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:54,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:54,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T16:52:54,742 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/bb37cf09a16a4f15896ac50f8abd1908 2024-11-20T16:52:54,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/260649ca8cac4860968ff6e284de539e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/260649ca8cac4860968ff6e284de539e 2024-11-20T16:52:54,758 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/260649ca8cac4860968ff6e284de539e, entries=150, sequenceid=324, filesize=30.5 K 2024-11-20T16:52:54,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/e0e7f79d4d0447bd9a4e975ebf95efb2 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/e0e7f79d4d0447bd9a4e975ebf95efb2 2024-11-20T16:52:54,764 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/e0e7f79d4d0447bd9a4e975ebf95efb2, entries=150, sequenceid=324, filesize=12.0 K 2024-11-20T16:52:54,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/bb37cf09a16a4f15896ac50f8abd1908 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/bb37cf09a16a4f15896ac50f8abd1908 2024-11-20T16:52:54,773 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/bb37cf09a16a4f15896ac50f8abd1908, entries=150, sequenceid=324, filesize=12.0 K 2024-11-20T16:52:54,774 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 94c122f231a59b575ec7f40dae2c00f4 in 583ms, sequenceid=324, compaction requested=false 2024-11-20T16:52:54,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:54,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:54,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-20T16:52:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-20T16:52:54,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-20T16:52:54,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2030 sec 2024-11-20T16:52:54,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.2140 sec 2024-11-20T16:52:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:55,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121574324, max timestamp=1732121575008Num uniques 859; 2024-11-20T16:52:55,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121574324, max timestamp=1732121575008Num uniques 859; 2024-11-20T16:52:55,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121574324, max timestamp=1732121575008Num uniques 859; 2024-11-20T16:52:55,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:52:55,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:52:55,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:52:55,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-20T16:52:55,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:55,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:55,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:55,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:55,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:55,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:55,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201b09407288a841d8ad664cf9d1d721a7_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121575009/Put/seqid=0 2024-11-20T16:52:55,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742044_1220 (size=12454) 2024-11-20T16:52:55,051 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:55,062 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201b09407288a841d8ad664cf9d1d721a7_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201b09407288a841d8ad664cf9d1d721a7_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:55,067 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/e21ae777ddfb4f6d81c3666ba76c4293, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:55,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/e21ae777ddfb4f6d81c3666ba76c4293 is 175, key is test_row_0/A:col10/1732121575009/Put/seqid=0 2024-11-20T16:52:55,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121575029, max timestamp=1732121575076Num uniques 859; 2024-11-20T16:52:55,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:52:55,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121575029, max timestamp=1732121575076Num uniques 859; 2024-11-20T16:52:55,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:52:55,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732121575029, max timestamp=1732121575076Num uniques 859; 2024-11-20T16:52:55,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-20T16:52:55,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742045_1221 (size=31255) 2024-11-20T16:52:55,086 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/e21ae777ddfb4f6d81c3666ba76c4293 2024-11-20T16:52:55,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/face194c131b49309621b70e07375108 is 50, key is test_row_0/B:col10/1732121575009/Put/seqid=0 2024-11-20T16:52:55,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742046_1222 (size=12301) 2024-11-20T16:52:55,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/face194c131b49309621b70e07375108 2024-11-20T16:52:55,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732121575127, max timestamp=1732121575147Num uniques 1018; 2024-11-20T16:52:55,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-20T16:52:55,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732121575127, max timestamp=1732121575147Num uniques 1018; 2024-11-20T16:52:55,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-20T16:52:55,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732121575127, max timestamp=1732121575147Num uniques 1009; 2024-11-20T16:52:55,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=12268, suffix heap size=132904,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=268 2024-11-20T16:52:55,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/9d25a319e03f405496c3e543c15de5f8 is 50, key is test_row_0/C:col10/1732121575009/Put/seqid=0 2024-11-20T16:52:55,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742047_1223 (size=12301) 2024-11-20T16:52:55,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121635228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:55,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121635229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:55,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:55,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121635238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:55,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121635347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:55,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:55,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121635347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:55,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:55,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121635347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:55,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:55,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121635556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:55,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:55,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121635566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121635566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:55,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/9d25a319e03f405496c3e543c15de5f8 2024-11-20T16:52:55,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/e21ae777ddfb4f6d81c3666ba76c4293 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/e21ae777ddfb4f6d81c3666ba76c4293 2024-11-20T16:52:55,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/e21ae777ddfb4f6d81c3666ba76c4293, entries=150, sequenceid=348, filesize=30.5 K 2024-11-20T16:52:55,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/face194c131b49309621b70e07375108 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/face194c131b49309621b70e07375108 2024-11-20T16:52:55,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/face194c131b49309621b70e07375108, entries=150, sequenceid=348, filesize=12.0 K 2024-11-20T16:52:55,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/9d25a319e03f405496c3e543c15de5f8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/9d25a319e03f405496c3e543c15de5f8 2024-11-20T16:52:55,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/9d25a319e03f405496c3e543c15de5f8, entries=150, sequenceid=348, filesize=12.0 K 2024-11-20T16:52:55,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=147.60 KB/151140 for 94c122f231a59b575ec7f40dae2c00f4 in 610ms, sequenceid=348, compaction requested=true 2024-11-20T16:52:55,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:55,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:55,622 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:55,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:55,622 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:55,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:55,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:55,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:55,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:55,623 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:55,623 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:55,623 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/B is initiating minor compaction (all files) 2024-11-20T16:52:55,623 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/A is initiating minor compaction (all files) 2024-11-20T16:52:55,623 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/A in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:55,623 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/500e6a0a0ba841cd8eecb0999f62fad7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/260649ca8cac4860968ff6e284de539e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/e21ae777ddfb4f6d81c3666ba76c4293] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=92.2 K 2024-11-20T16:52:55,623 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:55,623 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/B in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:55,623 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/500e6a0a0ba841cd8eecb0999f62fad7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/260649ca8cac4860968ff6e284de539e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/e21ae777ddfb4f6d81c3666ba76c4293] 2024-11-20T16:52:55,624 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/45424478ab5045f6be398ac9d377bafa, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/e0e7f79d4d0447bd9a4e975ebf95efb2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/face194c131b49309621b70e07375108] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=36.7 K 2024-11-20T16:52:55,624 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 500e6a0a0ba841cd8eecb0999f62fad7, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732121571975 2024-11-20T16:52:55,624 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 45424478ab5045f6be398ac9d377bafa, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732121571975 2024-11-20T16:52:55,624 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 260649ca8cac4860968ff6e284de539e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732121573133 2024-11-20T16:52:55,624 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting e0e7f79d4d0447bd9a4e975ebf95efb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732121573133 2024-11-20T16:52:55,624 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting e21ae777ddfb4f6d81c3666ba76c4293, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732121574350 2024-11-20T16:52:55,625 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting face194c131b49309621b70e07375108, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732121574350 2024-11-20T16:52:55,633 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#B#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:55,634 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/5df02ce79aa1435b8f0d1afad8075a5e is 50, key is test_row_0/B:col10/1732121575009/Put/seqid=0 2024-11-20T16:52:55,637 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:55,651 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411206124f7a6468f42f8b301eee747dd2390_94c122f231a59b575ec7f40dae2c00f4 store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:55,653 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411206124f7a6468f42f8b301eee747dd2390_94c122f231a59b575ec7f40dae2c00f4, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:55,654 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206124f7a6468f42f8b301eee747dd2390_94c122f231a59b575ec7f40dae2c00f4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:55,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742048_1224 (size=13051) 2024-11-20T16:52:55,666 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/5df02ce79aa1435b8f0d1afad8075a5e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/5df02ce79aa1435b8f0d1afad8075a5e 2024-11-20T16:52:55,678 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/B of 94c122f231a59b575ec7f40dae2c00f4 into 5df02ce79aa1435b8f0d1afad8075a5e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:55,678 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:55,679 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/B, priority=13, startTime=1732121575622; duration=0sec 2024-11-20T16:52:55,679 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:55,679 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:B 2024-11-20T16:52:55,679 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:55,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742049_1225 (size=4469) 2024-11-20T16:52:55,686 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:55,686 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/C is initiating minor compaction (all files) 2024-11-20T16:52:55,686 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/C in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:55,687 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/8ca8f765bb2945ad88d38b34d455233b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/bb37cf09a16a4f15896ac50f8abd1908, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/9d25a319e03f405496c3e543c15de5f8] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=36.7 K 2024-11-20T16:52:55,687 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ca8f765bb2945ad88d38b34d455233b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732121571975 2024-11-20T16:52:55,688 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting bb37cf09a16a4f15896ac50f8abd1908, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732121573133 2024-11-20T16:52:55,688 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d25a319e03f405496c3e543c15de5f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732121574350 2024-11-20T16:52:55,697 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#C#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:55,697 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/cf3d7e7991c64e2e8a57b469c08b53db is 50, key is test_row_0/C:col10/1732121575009/Put/seqid=0 2024-11-20T16:52:55,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742050_1226 (size=13051) 2024-11-20T16:52:55,710 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/cf3d7e7991c64e2e8a57b469c08b53db as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/cf3d7e7991c64e2e8a57b469c08b53db 2024-11-20T16:52:55,715 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/C of 94c122f231a59b575ec7f40dae2c00f4 into cf3d7e7991c64e2e8a57b469c08b53db(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:55,716 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:55,716 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/C, priority=13, startTime=1732121575622; duration=0sec 2024-11-20T16:52:55,716 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:55,716 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:C 2024-11-20T16:52:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T16:52:55,717 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:55,717 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-20T16:52:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T16:52:55,720 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:55,721 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:55,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T16:52:55,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732121575208, max timestamp=1732121575872Num uniques 1127; 2024-11-20T16:52:55,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-20T16:52:55,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T16:52:55,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:55,877 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=105.11 KB heapSize=291.28 KB 2024-11-20T16:52:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:55,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:55,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:52:55,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:55,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T16:52:55,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:55,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T16:52:55,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:55,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112088010b2b323f47cd8d275a9b0fe1fe2d_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121575236/Put/seqid=0 2024-11-20T16:52:55,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742051_1227 (size=12454) 2024-11-20T16:52:55,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:55,912 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112088010b2b323f47cd8d275a9b0fe1fe2d_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112088010b2b323f47cd8d275a9b0fe1fe2d_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:55,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121635911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:55,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/0f6d161b903c42ac8347a524a09fabbe, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:55,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:55,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/0f6d161b903c42ac8347a524a09fabbe is 175, key is test_row_0/A:col10/1732121575236/Put/seqid=0 2024-11-20T16:52:55,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121635912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:55,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121635914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:55,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742052_1228 (size=31255) 2024-11-20T16:52:55,937 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=405, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/0f6d161b903c42ac8347a524a09fabbe 2024-11-20T16:52:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/80bd8bdba6fc4b72b305c945f6a1e744 is 50, key is test_row_0/B:col10/1732121575236/Put/seqid=0 2024-11-20T16:52:56,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742053_1229 (size=12301) 2024-11-20T16:52:56,006 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/80bd8bdba6fc4b72b305c945f6a1e744 2024-11-20T16:52:56,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/64b8fa1df23440df9f0abd24e8f42c52 is 50, key is test_row_0/C:col10/1732121575236/Put/seqid=0 2024-11-20T16:52:56,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121636017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:56,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121636026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121636026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T16:52:56,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742054_1230 (size=12301) 2024-11-20T16:52:56,043 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/64b8fa1df23440df9f0abd24e8f42c52 2024-11-20T16:52:56,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/0f6d161b903c42ac8347a524a09fabbe as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0f6d161b903c42ac8347a524a09fabbe 2024-11-20T16:52:56,055 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0f6d161b903c42ac8347a524a09fabbe, entries=150, sequenceid=405, filesize=30.5 K 2024-11-20T16:52:56,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/80bd8bdba6fc4b72b305c945f6a1e744 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/80bd8bdba6fc4b72b305c945f6a1e744 2024-11-20T16:52:56,063 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/80bd8bdba6fc4b72b305c945f6a1e744, entries=150, sequenceid=405, filesize=12.0 K 2024-11-20T16:52:56,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/64b8fa1df23440df9f0abd24e8f42c52 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/64b8fa1df23440df9f0abd24e8f42c52 2024-11-20T16:52:56,069 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/64b8fa1df23440df9f0abd24e8f42c52, entries=150, sequenceid=405, filesize=12.0 K 2024-11-20T16:52:56,070 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~111.82 KB/114500, heapSize ~311.63 KB/319104, currentSize=73.80 KB/75570 for 94c122f231a59b575ec7f40dae2c00f4 in 193ms, sequenceid=405, compaction requested=false 2024-11-20T16:52:56,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:56,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:56,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-20T16:52:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-20T16:52:56,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-20T16:52:56,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 351 msec 2024-11-20T16:52:56,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 358 msec 2024-11-20T16:52:56,088 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#A#compaction#187 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:56,088 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/e664616ab43f4d4397c829a1ad111247 is 175, key is test_row_0/A:col10/1732121575009/Put/seqid=0 2024-11-20T16:52:56,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742055_1231 (size=32005) 2024-11-20T16:52:56,143 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/e664616ab43f4d4397c829a1ad111247 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/e664616ab43f4d4397c829a1ad111247 2024-11-20T16:52:56,148 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/A of 94c122f231a59b575ec7f40dae2c00f4 into e664616ab43f4d4397c829a1ad111247(size=31.3 K), total size for store is 61.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:56,148 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:56,148 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/A, priority=13, startTime=1732121575622; duration=0sec 2024-11-20T16:52:56,148 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:56,148 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:A 2024-11-20T16:52:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:56,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T16:52:56,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:56,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:56,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:56,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:56,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:56,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:56,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f38598172cf2437ebe652a2047930f7c_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121575911/Put/seqid=0 2024-11-20T16:52:56,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742056_1232 (size=14994) 2024-11-20T16:52:56,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121636259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:56,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121636260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:56,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121636263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T16:52:56,346 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:56,346 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-20T16:52:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T16:52:56,348 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:56,349 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:56,349 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:56,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:56,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121636366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:56,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:56,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121636376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:56,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121636376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T16:52:56,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-20T16:52:56,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:56,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:56,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:56,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:56,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:56,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121636576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:56,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:56,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121636586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121636586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:56,651 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:56,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-20T16:52:56,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:56,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:56,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:56,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:56,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:56,657 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f38598172cf2437ebe652a2047930f7c_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f38598172cf2437ebe652a2047930f7c_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:56,659 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/73eeda4307f7426092e70b384c944334, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:56,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/73eeda4307f7426092e70b384c944334 is 175, key is test_row_0/A:col10/1732121575911/Put/seqid=0 2024-11-20T16:52:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T16:52:56,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742057_1233 (size=39949) 2024-11-20T16:52:56,676 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=421, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/73eeda4307f7426092e70b384c944334 2024-11-20T16:52:56,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/e54431d39318493d8d93be0f31a6e288 is 50, key is test_row_0/B:col10/1732121575911/Put/seqid=0 2024-11-20T16:52:56,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742058_1234 (size=12301) 2024-11-20T16:52:56,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/e54431d39318493d8d93be0f31a6e288 2024-11-20T16:52:56,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/0671bcd4fa0546ad9f820aa426efbd1a is 50, key is test_row_0/C:col10/1732121575911/Put/seqid=0 2024-11-20T16:52:56,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742059_1235 (size=12301) 2024-11-20T16:52:56,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-20T16:52:56,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:56,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:56,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:56,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:56,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:56,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:56,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121636886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:56,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:56,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121636896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:56,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:56,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121636896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:56,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:56,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-20T16:52:56,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:56,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:56,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:56,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:56,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T16:52:57,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-20T16:52:57,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:57,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:57,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:57,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:57,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:57,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/0671bcd4fa0546ad9f820aa426efbd1a 2024-11-20T16:52:57,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/73eeda4307f7426092e70b384c944334 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/73eeda4307f7426092e70b384c944334 2024-11-20T16:52:57,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/73eeda4307f7426092e70b384c944334, entries=200, sequenceid=421, filesize=39.0 K 2024-11-20T16:52:57,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/e54431d39318493d8d93be0f31a6e288 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/e54431d39318493d8d93be0f31a6e288 2024-11-20T16:52:57,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/e54431d39318493d8d93be0f31a6e288, entries=150, sequenceid=421, filesize=12.0 K 2024-11-20T16:52:57,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/0671bcd4fa0546ad9f820aa426efbd1a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/0671bcd4fa0546ad9f820aa426efbd1a 2024-11-20T16:52:57,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/0671bcd4fa0546ad9f820aa426efbd1a, entries=150, sequenceid=421, filesize=12.0 K 2024-11-20T16:52:57,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 94c122f231a59b575ec7f40dae2c00f4 in 929ms, sequenceid=421, compaction requested=true 2024-11-20T16:52:57,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:57,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:57,168 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:57,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:57,168 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:57,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:57,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:57,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:57,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:57,169 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:57,169 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/A is initiating minor compaction (all files) 2024-11-20T16:52:57,169 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/A in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:57,169 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/e664616ab43f4d4397c829a1ad111247, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0f6d161b903c42ac8347a524a09fabbe, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/73eeda4307f7426092e70b384c944334] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=100.8 K 2024-11-20T16:52:57,169 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:57,169 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/e664616ab43f4d4397c829a1ad111247, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0f6d161b903c42ac8347a524a09fabbe, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/73eeda4307f7426092e70b384c944334] 2024-11-20T16:52:57,169 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:57,169 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/B is initiating minor compaction (all files) 2024-11-20T16:52:57,170 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/B in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:57,170 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/5df02ce79aa1435b8f0d1afad8075a5e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/80bd8bdba6fc4b72b305c945f6a1e744, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/e54431d39318493d8d93be0f31a6e288] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=36.8 K 2024-11-20T16:52:57,170 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting e664616ab43f4d4397c829a1ad111247, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732121574350 2024-11-20T16:52:57,170 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 5df02ce79aa1435b8f0d1afad8075a5e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732121574350 2024-11-20T16:52:57,170 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f6d161b903c42ac8347a524a09fabbe, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732121575227 2024-11-20T16:52:57,171 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 80bd8bdba6fc4b72b305c945f6a1e744, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732121575227 2024-11-20T16:52:57,171 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73eeda4307f7426092e70b384c944334, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732121575908 2024-11-20T16:52:57,171 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting e54431d39318493d8d93be0f31a6e288, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732121575910 2024-11-20T16:52:57,185 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#B#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:57,185 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/7fe13f3adf4e4d1da71a12f02f31ada2 is 50, key is test_row_0/B:col10/1732121575911/Put/seqid=0 2024-11-20T16:52:57,191 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:57,212 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120082e19af22e64853abefd7613a4035f8_94c122f231a59b575ec7f40dae2c00f4 store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:57,214 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120082e19af22e64853abefd7613a4035f8_94c122f231a59b575ec7f40dae2c00f4, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:57,214 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120082e19af22e64853abefd7613a4035f8_94c122f231a59b575ec7f40dae2c00f4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:57,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742060_1236 (size=13153) 2024-11-20T16:52:57,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742061_1237 (size=4469) 2024-11-20T16:52:57,259 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#A#compaction#196 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:57,260 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/837ffa3bb1f34e2e937b20435c61b705 is 175, key is test_row_0/A:col10/1732121575911/Put/seqid=0 2024-11-20T16:52:57,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-20T16:52:57,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:57,277 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T16:52:57,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:57,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:57,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:57,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:57,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:57,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:57,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120568972535e0b47dfb5651f7212e66940_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121576259/Put/seqid=0 2024-11-20T16:52:57,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742062_1238 (size=32107) 2024-11-20T16:52:57,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742063_1239 (size=12454) 2024-11-20T16:52:57,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:57,328 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120568972535e0b47dfb5651f7212e66940_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120568972535e0b47dfb5651f7212e66940_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:57,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/9748399cec2740d78426ff0b4245ef24, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:57,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/9748399cec2740d78426ff0b4245ef24 is 175, key is test_row_0/A:col10/1732121576259/Put/seqid=0 2024-11-20T16:52:57,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742064_1240 (size=31255) 2024-11-20T16:52:57,350 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=438, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/9748399cec2740d78426ff0b4245ef24 2024-11-20T16:52:57,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/457c79ae0105462ca860925b4e0d6738 is 50, key is test_row_0/B:col10/1732121576259/Put/seqid=0 2024-11-20T16:52:57,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742065_1241 (size=12301) 2024-11-20T16:52:57,393 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/457c79ae0105462ca860925b4e0d6738 2024-11-20T16:52:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:57,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:57,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/a26922420d404a7abf34fb03e4f944ac is 50, key is test_row_0/C:col10/1732121576259/Put/seqid=0 2024-11-20T16:52:57,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121637437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:57,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121637447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:57,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:57,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121637450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:57,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742066_1242 (size=12301) 2024-11-20T16:52:57,479 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/a26922420d404a7abf34fb03e4f944ac 2024-11-20T16:52:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T16:52:57,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/9748399cec2740d78426ff0b4245ef24 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/9748399cec2740d78426ff0b4245ef24 2024-11-20T16:52:57,496 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/9748399cec2740d78426ff0b4245ef24, entries=150, sequenceid=438, filesize=30.5 K 2024-11-20T16:52:57,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/457c79ae0105462ca860925b4e0d6738 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/457c79ae0105462ca860925b4e0d6738 2024-11-20T16:52:57,505 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/457c79ae0105462ca860925b4e0d6738, entries=150, sequenceid=438, filesize=12.0 K 2024-11-20T16:52:57,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/a26922420d404a7abf34fb03e4f944ac as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/a26922420d404a7abf34fb03e4f944ac 2024-11-20T16:52:57,521 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/a26922420d404a7abf34fb03e4f944ac, entries=150, sequenceid=438, filesize=12.0 K 2024-11-20T16:52:57,522 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 94c122f231a59b575ec7f40dae2c00f4 in 245ms, sequenceid=438, compaction requested=true 2024-11-20T16:52:57,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:57,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:57,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-20T16:52:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-20T16:52:57,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-20T16:52:57,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1770 sec 2024-11-20T16:52:57,530 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.1830 sec 2024-11-20T16:52:57,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T16:52:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:57,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:57,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:57,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:57,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:57,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:57,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:57,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112046837b5abe8646c59f542f506e8e72bb_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121577567/Put/seqid=0 2024-11-20T16:52:57,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121637619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:57,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:57,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121637621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:57,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121637622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:57,646 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/7fe13f3adf4e4d1da71a12f02f31ada2 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/7fe13f3adf4e4d1da71a12f02f31ada2 2024-11-20T16:52:57,655 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/B of 94c122f231a59b575ec7f40dae2c00f4 into 7fe13f3adf4e4d1da71a12f02f31ada2(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:57,655 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:57,655 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/B, priority=13, startTime=1732121577168; duration=0sec 2024-11-20T16:52:57,656 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:57,656 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:B 2024-11-20T16:52:57,656 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:52:57,658 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:52:57,658 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/C is initiating minor compaction (all files) 2024-11-20T16:52:57,658 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/C in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:57,658 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/cf3d7e7991c64e2e8a57b469c08b53db, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/64b8fa1df23440df9f0abd24e8f42c52, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/0671bcd4fa0546ad9f820aa426efbd1a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/a26922420d404a7abf34fb03e4f944ac] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=48.8 K 2024-11-20T16:52:57,659 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting cf3d7e7991c64e2e8a57b469c08b53db, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732121574350 2024-11-20T16:52:57,659 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 64b8fa1df23440df9f0abd24e8f42c52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732121575227 2024-11-20T16:52:57,660 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 0671bcd4fa0546ad9f820aa426efbd1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732121575910 2024-11-20T16:52:57,660 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting a26922420d404a7abf34fb03e4f944ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732121576249 2024-11-20T16:52:57,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742067_1243 (size=17534) 2024-11-20T16:52:57,678 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#C#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:57,678 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/0a69267ca41f4a179baca16de3d94a98 is 50, key is test_row_0/C:col10/1732121576259/Put/seqid=0 2024-11-20T16:52:57,724 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/837ffa3bb1f34e2e937b20435c61b705 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/837ffa3bb1f34e2e937b20435c61b705 2024-11-20T16:52:57,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121637727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:57,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:57,735 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/A of 94c122f231a59b575ec7f40dae2c00f4 into 837ffa3bb1f34e2e937b20435c61b705(size=31.4 K), total size for store is 61.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:57,735 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:57,735 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/A, priority=13, startTime=1732121577167; duration=0sec 2024-11-20T16:52:57,735 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:57,735 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:A 2024-11-20T16:52:57,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742068_1244 (size=13187) 2024-11-20T16:52:57,746 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/0a69267ca41f4a179baca16de3d94a98 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/0a69267ca41f4a179baca16de3d94a98 2024-11-20T16:52:57,753 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/C of 94c122f231a59b575ec7f40dae2c00f4 into 0a69267ca41f4a179baca16de3d94a98(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:57,753 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:57,753 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/C, priority=12, startTime=1732121577168; duration=0sec 2024-11-20T16:52:57,753 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:57,753 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:C 2024-11-20T16:52:57,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:57,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121637736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:57,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:57,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121637736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:57,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:57,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121637936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:57,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:57,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121637966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:57,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:57,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:57,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121637967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:57,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:57,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:58,074 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:58,080 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112046837b5abe8646c59f542f506e8e72bb_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112046837b5abe8646c59f542f506e8e72bb_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:58,081 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/0b22cbc42c67425bbb5ba4de926cfc8d, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:58,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/0b22cbc42c67425bbb5ba4de926cfc8d is 175, key is test_row_0/A:col10/1732121577567/Put/seqid=0 2024-11-20T16:52:58,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742069_1245 (size=48639) 2024-11-20T16:52:58,117 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=455, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/0b22cbc42c67425bbb5ba4de926cfc8d 2024-11-20T16:52:58,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/0d31c520bda342a19b3cb006db310591 is 50, key is test_row_0/B:col10/1732121577567/Put/seqid=0 2024-11-20T16:52:58,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742070_1246 (size=12301) 2024-11-20T16:52:58,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/0d31c520bda342a19b3cb006db310591 2024-11-20T16:52:58,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/26aa585f23b846c5a9077c6f4f855e9d is 50, key is test_row_0/C:col10/1732121577567/Put/seqid=0 2024-11-20T16:52:58,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742071_1247 (size=12301) 2024-11-20T16:52:58,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121638246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:58,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121638276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:58,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:58,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:58,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:58,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121638286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:58,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:58,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:58,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T16:52:58,496 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:52:58,497 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:52:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-20T16:52:58,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T16:52:58,502 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:52:58,503 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:52:58,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:52:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T16:52:58,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/26aa585f23b846c5a9077c6f4f855e9d 2024-11-20T16:52:58,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/0b22cbc42c67425bbb5ba4de926cfc8d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0b22cbc42c67425bbb5ba4de926cfc8d 2024-11-20T16:52:58,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T16:52:58,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:58,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:58,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:58,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:58,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:58,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:58,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0b22cbc42c67425bbb5ba4de926cfc8d, entries=250, sequenceid=455, filesize=47.5 K 2024-11-20T16:52:58,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/0d31c520bda342a19b3cb006db310591 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/0d31c520bda342a19b3cb006db310591 2024-11-20T16:52:58,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/0d31c520bda342a19b3cb006db310591, entries=150, sequenceid=455, filesize=12.0 K 2024-11-20T16:52:58,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/26aa585f23b846c5a9077c6f4f855e9d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/26aa585f23b846c5a9077c6f4f855e9d 2024-11-20T16:52:58,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/26aa585f23b846c5a9077c6f4f855e9d, entries=150, sequenceid=455, filesize=12.0 K 2024-11-20T16:52:58,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 94c122f231a59b575ec7f40dae2c00f4 in 1121ms, sequenceid=455, compaction requested=true 2024-11-20T16:52:58,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:58,689 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:58,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:58,690 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 112001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:58,690 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/A is initiating minor compaction (all files) 2024-11-20T16:52:58,690 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/A in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:58,691 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/837ffa3bb1f34e2e937b20435c61b705, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/9748399cec2740d78426ff0b4245ef24, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0b22cbc42c67425bbb5ba4de926cfc8d] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=109.4 K 2024-11-20T16:52:58,691 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:58,691 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/837ffa3bb1f34e2e937b20435c61b705, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/9748399cec2740d78426ff0b4245ef24, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0b22cbc42c67425bbb5ba4de926cfc8d] 2024-11-20T16:52:58,691 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 837ffa3bb1f34e2e937b20435c61b705, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732121575910 2024-11-20T16:52:58,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:58,691 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:58,692 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9748399cec2740d78426ff0b4245ef24, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732121576249 2024-11-20T16:52:58,692 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b22cbc42c67425bbb5ba4de926cfc8d, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1732121577423 2024-11-20T16:52:58,693 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:58,693 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/B is initiating minor compaction (all files) 2024-11-20T16:52:58,693 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/B in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:58,693 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/7fe13f3adf4e4d1da71a12f02f31ada2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/457c79ae0105462ca860925b4e0d6738, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/0d31c520bda342a19b3cb006db310591] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=36.9 K 2024-11-20T16:52:58,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:58,694 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fe13f3adf4e4d1da71a12f02f31ada2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732121575910 2024-11-20T16:52:58,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:58,694 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 457c79ae0105462ca860925b4e0d6738, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732121576249 2024-11-20T16:52:58,695 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d31c520bda342a19b3cb006db310591, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1732121577445 2024-11-20T16:52:58,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:58,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:58,711 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:58,720 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#B#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:58,721 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/12e6ddfd0f9049a894d86235bb553988 is 50, key is test_row_0/B:col10/1732121577567/Put/seqid=0 2024-11-20T16:52:58,732 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120a2e597f7866f484db1ba5beaef027d72_94c122f231a59b575ec7f40dae2c00f4 store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:58,734 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120a2e597f7866f484db1ba5beaef027d72_94c122f231a59b575ec7f40dae2c00f4, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:58,735 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a2e597f7866f484db1ba5beaef027d72_94c122f231a59b575ec7f40dae2c00f4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:58,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T16:52:58,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:58,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:58,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:58,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:58,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:58,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:58,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742072_1248 (size=13255) 2024-11-20T16:52:58,794 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/12e6ddfd0f9049a894d86235bb553988 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/12e6ddfd0f9049a894d86235bb553988 2024-11-20T16:52:58,803 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/B of 94c122f231a59b575ec7f40dae2c00f4 into 12e6ddfd0f9049a894d86235bb553988(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:58,803 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:58,803 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/B, priority=13, startTime=1732121578691; duration=0sec 2024-11-20T16:52:58,803 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:58,803 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:B 2024-11-20T16:52:58,803 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T16:52:58,805 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T16:52:58,805 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T16:52:58,805 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. because compaction request was cancelled 2024-11-20T16:52:58,805 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:C 2024-11-20T16:52:58,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742073_1249 (size=4469) 2024-11-20T16:52:58,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T16:52:58,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:58,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:58,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:58,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:58,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T16:52:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:58,820 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#A#compaction#204 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:58,820 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/2e186cbacfe242aa83673342bd54951d is 175, key is test_row_0/A:col10/1732121577567/Put/seqid=0 2024-11-20T16:52:58,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b9fdc3af8dce4715914006178acf1964_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121577620/Put/seqid=0 2024-11-20T16:52:58,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742074_1250 (size=32209) 2024-11-20T16:52:58,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742075_1251 (size=14994) 2024-11-20T16:52:58,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121638911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:58,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:58,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121638911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121638911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:58,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T16:52:58,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:58,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:58,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:58,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:58,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:59,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121639036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:59,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121639036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:59,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121639036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:59,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:59,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:59,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T16:52:59,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:59,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:59,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:59,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T16:52:59,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T16:52:59,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:59,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:59,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:59,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121639286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:59,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121639286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:59,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121639291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:59,299 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:52:59,305 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b9fdc3af8dce4715914006178acf1964_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b9fdc3af8dce4715914006178acf1964_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:59,306 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/2e186cbacfe242aa83673342bd54951d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/2e186cbacfe242aa83673342bd54951d 2024-11-20T16:52:59,307 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/9f94e3775a0048a5a675cacfd2f5d4fc, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:52:59,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/9f94e3775a0048a5a675cacfd2f5d4fc is 175, key is test_row_0/A:col10/1732121577620/Put/seqid=0 2024-11-20T16:52:59,317 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/A of 94c122f231a59b575ec7f40dae2c00f4 into 2e186cbacfe242aa83673342bd54951d(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:59,317 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:59,317 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/A, priority=13, startTime=1732121578689; duration=0sec 2024-11-20T16:52:59,317 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:59,317 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:A 2024-11-20T16:52:59,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742076_1252 (size=39949) 2024-11-20T16:52:59,328 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=474, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/9f94e3775a0048a5a675cacfd2f5d4fc 2024-11-20T16:52:59,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/51d7bad1b8a04a69b5a4b7aba99d5bda is 50, key is test_row_0/B:col10/1732121577620/Put/seqid=0 2024-11-20T16:52:59,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742077_1253 (size=12301) 2024-11-20T16:52:59,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/51d7bad1b8a04a69b5a4b7aba99d5bda 2024-11-20T16:52:59,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/1a38f9c4375f4cac983880b5f08faf87 is 50, key is test_row_0/C:col10/1732121577620/Put/seqid=0 2024-11-20T16:52:59,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T16:52:59,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:59,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:59,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:59,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742078_1254 (size=12301) 2024-11-20T16:52:59,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/1a38f9c4375f4cac983880b5f08faf87 2024-11-20T16:52:59,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/9f94e3775a0048a5a675cacfd2f5d4fc as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/9f94e3775a0048a5a675cacfd2f5d4fc 2024-11-20T16:52:59,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/9f94e3775a0048a5a675cacfd2f5d4fc, entries=200, sequenceid=474, filesize=39.0 K 2024-11-20T16:52:59,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/51d7bad1b8a04a69b5a4b7aba99d5bda as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/51d7bad1b8a04a69b5a4b7aba99d5bda 2024-11-20T16:52:59,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/51d7bad1b8a04a69b5a4b7aba99d5bda, entries=150, sequenceid=474, filesize=12.0 K 2024-11-20T16:52:59,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/1a38f9c4375f4cac983880b5f08faf87 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/1a38f9c4375f4cac983880b5f08faf87 2024-11-20T16:52:59,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/1a38f9c4375f4cac983880b5f08faf87, entries=150, sequenceid=474, filesize=12.0 K 2024-11-20T16:52:59,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 94c122f231a59b575ec7f40dae2c00f4 in 750ms, sequenceid=474, compaction requested=true 2024-11-20T16:52:59,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:59,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:52:59,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:59,513 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T16:52:59,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:52:59,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:52:59,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:52:59,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T16:52:59,513 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:52:59,514 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T16:52:59,514 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T16:52:59,514 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. because compaction request was cancelled 2024-11-20T16:52:59,514 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:A 2024-11-20T16:52:59,514 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T16:52:59,514 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:52:59,515 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/C is initiating minor compaction (all files) 2024-11-20T16:52:59,515 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/C in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:59,515 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/0a69267ca41f4a179baca16de3d94a98, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/26aa585f23b846c5a9077c6f4f855e9d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/1a38f9c4375f4cac983880b5f08faf87] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=36.9 K 2024-11-20T16:52:59,516 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a69267ca41f4a179baca16de3d94a98, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732121576249 2024-11-20T16:52:59,516 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T16:52:59,516 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T16:52:59,516 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. because compaction request was cancelled 2024-11-20T16:52:59,516 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:B 2024-11-20T16:52:59,516 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 26aa585f23b846c5a9077c6f4f855e9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1732121577445 2024-11-20T16:52:59,517 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a38f9c4375f4cac983880b5f08faf87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1732121577617 2024-11-20T16:52:59,532 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#C#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:52:59,533 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/823236e23a2042b098b5294ed9865ba8 is 50, key is test_row_0/C:col10/1732121577620/Put/seqid=0 2024-11-20T16:52:59,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742079_1255 (size=13289) 2024-11-20T16:52:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:52:59,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T16:52:59,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T16:52:59,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:52:59,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:59,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:59,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:52:59,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:59,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:59,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:52:59,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:59,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:52:59,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,604 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/823236e23a2042b098b5294ed9865ba8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/823236e23a2042b098b5294ed9865ba8 2024-11-20T16:52:59,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,615 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/C of 94c122f231a59b575ec7f40dae2c00f4 into 823236e23a2042b098b5294ed9865ba8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:52:59,615 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:52:59,615 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/C, priority=13, startTime=1732121579513; duration=0sec 2024-11-20T16:52:59,615 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:52:59,615 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:C 2024-11-20T16:52:59,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201585d6a800af4a88b1fa1c86ca857a28_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121578885/Put/seqid=0 2024-11-20T16:52:59,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T16:52:59,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:59,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121639653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:59,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:59,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121639664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:59,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:59,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121639665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:59,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:59,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:59,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742080_1256 (size=14994) 2024-11-20T16:52:59,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T16:52:59,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:59,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:59,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:59,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121639776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:59,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:59,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121639776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:59,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121639786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:59,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:59,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T16:52:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:52:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:52:59,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:52:59,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45304 deadline: 1732121639936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:59,939 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 94c122f231a59b575ec7f40dae2c00f4 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:52:59,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:59,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121639986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:59,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:59,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121639987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:59,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45236 deadline: 1732121639996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:59,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:52:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121639996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:52:59,997 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 94c122f231a59b575ec7f40dae2c00f4 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18299 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:52:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:52:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:52:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:00,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T16:53:00,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:53:00,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:53:00,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:53:00,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:00,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:00,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:00,132 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:00,140 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201585d6a800af4a88b1fa1c86ca857a28_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201585d6a800af4a88b1fa1c86ca857a28_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:00,143 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/b4ecf440859c4f3089768802a9f457ef, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:53:00,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/b4ecf440859c4f3089768802a9f457ef is 175, key is test_row_0/A:col10/1732121578885/Put/seqid=0 2024-11-20T16:53:00,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742081_1257 (size=39949) 2024-11-20T16:53:00,151 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=493, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/b4ecf440859c4f3089768802a9f457ef 2024-11-20T16:53:00,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/d3cf9ba777c3483da862ae7875a846ba is 50, key is test_row_0/B:col10/1732121578885/Put/seqid=0 2024-11-20T16:53:00,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T16:53:00,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:53:00,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:53:00,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:53:00,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:00,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:00,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742082_1258 (size=12301) 2024-11-20T16:53:00,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/d3cf9ba777c3483da862ae7875a846ba 2024-11-20T16:53:00,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/a3ed876167d8465c9a399441a3f59fdc is 50, key is test_row_0/C:col10/1732121578885/Put/seqid=0 2024-11-20T16:53:00,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742083_1259 (size=12301) 2024-11-20T16:53:00,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/a3ed876167d8465c9a399441a3f59fdc 2024-11-20T16:53:00,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/b4ecf440859c4f3089768802a9f457ef as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b4ecf440859c4f3089768802a9f457ef 2024-11-20T16:53:00,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:00,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121640306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121640306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:00,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:00,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:00,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:00,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:00,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121640306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:00,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b4ecf440859c4f3089768802a9f457ef, entries=200, sequenceid=493, filesize=39.0 K 2024-11-20T16:53:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:00,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/d3cf9ba777c3483da862ae7875a846ba as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/d3cf9ba777c3483da862ae7875a846ba 2024-11-20T16:53:00,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/d3cf9ba777c3483da862ae7875a846ba, entries=150, sequenceid=493, filesize=12.0 K 2024-11-20T16:53:00,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/a3ed876167d8465c9a399441a3f59fdc as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/a3ed876167d8465c9a399441a3f59fdc 2024-11-20T16:53:00,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/a3ed876167d8465c9a399441a3f59fdc, entries=150, sequenceid=493, filesize=12.0 K 2024-11-20T16:53:00,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 94c122f231a59b575ec7f40dae2c00f4 in 737ms, sequenceid=493, compaction requested=true 2024-11-20T16:53:00,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:53:00,337 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:00,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:53:00,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:00,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:53:00,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:00,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:53:00,338 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:00,338 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 112107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:00,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:00,339 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/A is initiating minor compaction (all files) 2024-11-20T16:53:00,339 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/A in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:53:00,339 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/2e186cbacfe242aa83673342bd54951d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/9f94e3775a0048a5a675cacfd2f5d4fc, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b4ecf440859c4f3089768802a9f457ef] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=109.5 K 2024-11-20T16:53:00,339 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:53:00,339 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/2e186cbacfe242aa83673342bd54951d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/9f94e3775a0048a5a675cacfd2f5d4fc, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b4ecf440859c4f3089768802a9f457ef] 2024-11-20T16:53:00,339 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:00,339 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e186cbacfe242aa83673342bd54951d, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1732121577445 2024-11-20T16:53:00,339 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/B is initiating minor compaction (all files) 2024-11-20T16:53:00,339 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/B in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:53:00,339 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/12e6ddfd0f9049a894d86235bb553988, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/51d7bad1b8a04a69b5a4b7aba99d5bda, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/d3cf9ba777c3483da862ae7875a846ba] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=37.0 K 2024-11-20T16:53:00,342 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f94e3775a0048a5a675cacfd2f5d4fc, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1732121577617 2024-11-20T16:53:00,342 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 12e6ddfd0f9049a894d86235bb553988, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1732121577445 2024-11-20T16:53:00,342 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 51d7bad1b8a04a69b5a4b7aba99d5bda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1732121577617 2024-11-20T16:53:00,342 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4ecf440859c4f3089768802a9f457ef, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732121578832 2024-11-20T16:53:00,342 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting d3cf9ba777c3483da862ae7875a846ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732121578832 2024-11-20T16:53:00,365 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:53:00,367 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#B#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:00,368 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/0a666df3b4104d7f81c8f476605e6dae is 50, key is test_row_0/B:col10/1732121578885/Put/seqid=0 2024-11-20T16:53:00,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T16:53:00,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:53:00,376 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T16:53:00,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:53:00,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:00,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:53:00,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:00,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:53:00,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:00,377 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120e7f45fb57b4f4e33887edf3098907057_94c122f231a59b575ec7f40dae2c00f4 store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:53:00,379 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120e7f45fb57b4f4e33887edf3098907057_94c122f231a59b575ec7f40dae2c00f4, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:53:00,379 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e7f45fb57b4f4e33887edf3098907057_94c122f231a59b575ec7f40dae2c00f4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:53:00,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742084_1260 (size=13357) 2024-11-20T16:53:00,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e8064c236c504326a74743fd8488cf88_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121579633/Put/seqid=0 2024-11-20T16:53:00,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742085_1261 (size=4469) 2024-11-20T16:53:00,470 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#A#compaction#214 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:00,471 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/f9ed673fae46490bbb69dc0f8c33f9fc is 175, key is test_row_0/A:col10/1732121578885/Put/seqid=0 2024-11-20T16:53:00,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742086_1262 (size=12454) 2024-11-20T16:53:00,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742087_1263 (size=32311) 2024-11-20T16:53:00,532 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/f9ed673fae46490bbb69dc0f8c33f9fc as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f9ed673fae46490bbb69dc0f8c33f9fc 2024-11-20T16:53:00,543 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/A of 94c122f231a59b575ec7f40dae2c00f4 into f9ed673fae46490bbb69dc0f8c33f9fc(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:00,543 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:53:00,543 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/A, priority=13, startTime=1732121580337; duration=0sec 2024-11-20T16:53:00,543 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:00,543 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:A 2024-11-20T16:53:00,543 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T16:53:00,544 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T16:53:00,545 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T16:53:00,545 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. because compaction request was cancelled 2024-11-20T16:53:00,545 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:C 2024-11-20T16:53:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T16:53:00,826 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/0a666df3b4104d7f81c8f476605e6dae as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/0a666df3b4104d7f81c8f476605e6dae 2024-11-20T16:53:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:00,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. as already flushing 2024-11-20T16:53:00,833 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/B of 94c122f231a59b575ec7f40dae2c00f4 into 0a666df3b4104d7f81c8f476605e6dae(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:00,833 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:53:00,833 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/B, priority=13, startTime=1732121580338; duration=0sec 2024-11-20T16:53:00,833 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:00,833 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:B 2024-11-20T16:53:00,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:00,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121640866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:00,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121640875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:00,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121640875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:00,926 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e8064c236c504326a74743fd8488cf88_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e8064c236c504326a74743fd8488cf88_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:00,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/ac8149d44e704e4293364e969904c85c, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:53:00,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/ac8149d44e704e4293364e969904c85c is 175, key is test_row_0/A:col10/1732121579633/Put/seqid=0 2024-11-20T16:53:00,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742088_1264 (size=31255) 2024-11-20T16:53:00,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:00,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121640986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121640986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:00,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121640986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:01,167 INFO [Thread-681 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-681. 2024-11-20T16:53:01,167 DEBUG [Thread-681 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:01,167 DEBUG [Thread-681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:01,168 DEBUG [Thread-681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:01,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:01,171 INFO [Thread-685 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-685. 2024-11-20T16:53:01,172 DEBUG [Thread-685 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:01,172 DEBUG [Thread-685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:01,172 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:01,172 DEBUG [Thread-685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:01,172 INFO [Thread-683 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-683. 2024-11-20T16:53:01,173 DEBUG [Thread-683 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:01,173 DEBUG [Thread-683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:01,173 DEBUG [Thread-683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:01,173 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:01,176 INFO [Thread-687 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-687. 2024-11-20T16:53:01,176 DEBUG [Thread-687 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:01,176 DEBUG [Thread-687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:01,176 DEBUG [Thread-687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:01,176 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:01,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:01,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121641196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121641196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:01,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121641207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:01,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:01,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:01,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:01,373 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=510, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/ac8149d44e704e4293364e969904c85c 2024-11-20T16:53:01,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/293ea847926f4ccb8c031a6158644a3f is 50, key is test_row_0/B:col10/1732121579633/Put/seqid=0 2024-11-20T16:53:01,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742089_1265 (size=12301) 2024-11-20T16:53:01,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:01,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45282 deadline: 1732121641506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732121641506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:01,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:01,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:01,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:01,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:01,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:01,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:01,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45258 deadline: 1732121641516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:01,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:01,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:01,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:01,786 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/293ea847926f4ccb8c031a6158644a3f 2024-11-20T16:53:01,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/3598bb7bbf5c430da2e2e54e26c01f9c is 50, key is test_row_0/C:col10/1732121579633/Put/seqid=0 2024-11-20T16:53:01,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742090_1266 (size=12301) 2024-11-20T16:53:01,813 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/3598bb7bbf5c430da2e2e54e26c01f9c 2024-11-20T16:53:01,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/ac8149d44e704e4293364e969904c85c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/ac8149d44e704e4293364e969904c85c 2024-11-20T16:53:01,826 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/ac8149d44e704e4293364e969904c85c, entries=150, sequenceid=510, filesize=30.5 K 2024-11-20T16:53:01,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/293ea847926f4ccb8c031a6158644a3f as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/293ea847926f4ccb8c031a6158644a3f 2024-11-20T16:53:01,833 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/293ea847926f4ccb8c031a6158644a3f, entries=150, sequenceid=510, filesize=12.0 K 2024-11-20T16:53:01,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/3598bb7bbf5c430da2e2e54e26c01f9c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/3598bb7bbf5c430da2e2e54e26c01f9c 2024-11-20T16:53:01,844 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/3598bb7bbf5c430da2e2e54e26c01f9c, entries=150, sequenceid=510, filesize=12.0 K 2024-11-20T16:53:01,845 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 94c122f231a59b575ec7f40dae2c00f4 in 1469ms, sequenceid=510, compaction requested=true 2024-11-20T16:53:01,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:53:01,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:53:01,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-20T16:53:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-20T16:53:01,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-20T16:53:01,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3440 sec 2024-11-20T16:53:01,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 3.3540 sec 2024-11-20T16:53:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:02,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T16:53:02,018 INFO [Thread-676 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-676. 2024-11-20T16:53:02,018 INFO [Thread-678 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-678. 2024-11-20T16:53:02,018 DEBUG [Thread-676 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:02,018 DEBUG [Thread-678 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:02,018 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:02,018 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:02,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:53:02,018 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:02,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:02,018 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:02,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:53:02,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:02,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:53:02,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:02,022 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:02,022 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:02,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b505bdbbd5824921be03f59d5a5c41ce_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_0/A:col10/1732121580864/Put/seqid=0 2024-11-20T16:53:02,028 INFO [Thread-672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-672. 2024-11-20T16:53:02,028 DEBUG [Thread-672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:02,028 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:02,028 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:02,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:02,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742091_1267 (size=12454) 2024-11-20T16:53:02,432 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:02,436 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b505bdbbd5824921be03f59d5a5c41ce_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b505bdbbd5824921be03f59d5a5c41ce_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:02,437 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/a45c59ba962242df919db257a10e4d2d, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:53:02,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/a45c59ba962242df919db257a10e4d2d is 175, key is test_row_0/A:col10/1732121580864/Put/seqid=0 2024-11-20T16:53:02,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742092_1268 (size=31255) 2024-11-20T16:53:02,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T16:53:02,667 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:02,845 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=531, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/a45c59ba962242df919db257a10e4d2d 2024-11-20T16:53:02,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/c630282f8b584546b2040f4545b5c12e is 50, key is test_row_0/B:col10/1732121580864/Put/seqid=0 2024-11-20T16:53:02,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742093_1269 (size=12301) 2024-11-20T16:53:03,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/c630282f8b584546b2040f4545b5c12e 2024-11-20T16:53:03,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/f59b4946f8544778a1cfcf3bfceb7e2a is 50, key is test_row_0/C:col10/1732121580864/Put/seqid=0 2024-11-20T16:53:03,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742094_1270 (size=12301) 2024-11-20T16:53:03,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/f59b4946f8544778a1cfcf3bfceb7e2a 2024-11-20T16:53:03,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/a45c59ba962242df919db257a10e4d2d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a45c59ba962242df919db257a10e4d2d 2024-11-20T16:53:03,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a45c59ba962242df919db257a10e4d2d, entries=150, sequenceid=531, filesize=30.5 K 2024-11-20T16:53:03,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/c630282f8b584546b2040f4545b5c12e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/c630282f8b584546b2040f4545b5c12e 2024-11-20T16:53:03,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/c630282f8b584546b2040f4545b5c12e, entries=150, sequenceid=531, filesize=12.0 K 2024-11-20T16:53:03,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/f59b4946f8544778a1cfcf3bfceb7e2a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/f59b4946f8544778a1cfcf3bfceb7e2a 2024-11-20T16:53:03,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/f59b4946f8544778a1cfcf3bfceb7e2a, entries=150, sequenceid=531, filesize=12.0 K 2024-11-20T16:53:03,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=6.71 KB/6870 for 94c122f231a59b575ec7f40dae2c00f4 in 1291ms, sequenceid=531, compaction requested=true 2024-11-20T16:53:03,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:53:03,310 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:03,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:53:03,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:03,310 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:03,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:53:03,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:03,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c122f231a59b575ec7f40dae2c00f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:53:03,311 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:03,311 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/A is initiating minor compaction (all files) 2024-11-20T16:53:03,311 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:03,311 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/B is initiating minor compaction (all files) 2024-11-20T16:53:03,311 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/A in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:53:03,311 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/B in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:53:03,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:03,311 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f9ed673fae46490bbb69dc0f8c33f9fc, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/ac8149d44e704e4293364e969904c85c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a45c59ba962242df919db257a10e4d2d] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=92.6 K 2024-11-20T16:53:03,311 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/0a666df3b4104d7f81c8f476605e6dae, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/293ea847926f4ccb8c031a6158644a3f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/c630282f8b584546b2040f4545b5c12e] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=37.1 K 2024-11-20T16:53:03,311 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:53:03,311 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f9ed673fae46490bbb69dc0f8c33f9fc, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/ac8149d44e704e4293364e969904c85c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a45c59ba962242df919db257a10e4d2d] 2024-11-20T16:53:03,312 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9ed673fae46490bbb69dc0f8c33f9fc, keycount=150, bloomtype=ROW, size=31.6 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732121578832 2024-11-20T16:53:03,312 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a666df3b4104d7f81c8f476605e6dae, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732121578832 2024-11-20T16:53:03,312 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 293ea847926f4ccb8c031a6158644a3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1732121579633 2024-11-20T16:53:03,312 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac8149d44e704e4293364e969904c85c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1732121579633 2024-11-20T16:53:03,312 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting c630282f8b584546b2040f4545b5c12e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1732121580864 2024-11-20T16:53:03,312 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting a45c59ba962242df919db257a10e4d2d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1732121580864 2024-11-20T16:53:03,323 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#B#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:03,324 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/6bb9b2409d084f30ad8fff9b320564ee is 50, key is test_row_0/B:col10/1732121580864/Put/seqid=0 2024-11-20T16:53:03,331 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:53:03,339 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112060f353a837cf4ec9965200af98ceae92_94c122f231a59b575ec7f40dae2c00f4 store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:53:03,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742095_1271 (size=13459) 2024-11-20T16:53:03,348 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/6bb9b2409d084f30ad8fff9b320564ee as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/6bb9b2409d084f30ad8fff9b320564ee 2024-11-20T16:53:03,357 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/B of 94c122f231a59b575ec7f40dae2c00f4 into 6bb9b2409d084f30ad8fff9b320564ee(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:03,357 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:53:03,357 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/B, priority=13, startTime=1732121583310; duration=0sec 2024-11-20T16:53:03,357 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:03,357 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:B 2024-11-20T16:53:03,357 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:53:03,359 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:53:03,359 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 94c122f231a59b575ec7f40dae2c00f4/C is initiating minor compaction (all files) 2024-11-20T16:53:03,359 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c122f231a59b575ec7f40dae2c00f4/C in TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:53:03,359 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/823236e23a2042b098b5294ed9865ba8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/a3ed876167d8465c9a399441a3f59fdc, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/3598bb7bbf5c430da2e2e54e26c01f9c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/f59b4946f8544778a1cfcf3bfceb7e2a] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp, totalSize=49.0 K 2024-11-20T16:53:03,360 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 823236e23a2042b098b5294ed9865ba8, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1732121577617 2024-11-20T16:53:03,360 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting a3ed876167d8465c9a399441a3f59fdc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732121578832 2024-11-20T16:53:03,361 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 3598bb7bbf5c430da2e2e54e26c01f9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1732121579633 2024-11-20T16:53:03,361 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting f59b4946f8544778a1cfcf3bfceb7e2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1732121580864 2024-11-20T16:53:03,376 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112060f353a837cf4ec9965200af98ceae92_94c122f231a59b575ec7f40dae2c00f4, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:53:03,377 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112060f353a837cf4ec9965200af98ceae92_94c122f231a59b575ec7f40dae2c00f4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:53:03,377 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#C#compaction#223 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:03,377 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/738b6882995b488980087e0aa362d34a is 50, key is test_row_0/C:col10/1732121580864/Put/seqid=0 2024-11-20T16:53:03,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742096_1272 (size=4469) 2024-11-20T16:53:03,395 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c122f231a59b575ec7f40dae2c00f4#A#compaction#222 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:03,396 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/52e6406ba447492387bd87b1c0ac5340 is 175, key is test_row_0/A:col10/1732121580864/Put/seqid=0 2024-11-20T16:53:03,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742097_1273 (size=13425) 2024-11-20T16:53:03,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742098_1274 (size=32413) 2024-11-20T16:53:03,439 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/738b6882995b488980087e0aa362d34a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/738b6882995b488980087e0aa362d34a 2024-11-20T16:53:03,439 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/52e6406ba447492387bd87b1c0ac5340 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/52e6406ba447492387bd87b1c0ac5340 2024-11-20T16:53:03,445 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/C of 94c122f231a59b575ec7f40dae2c00f4 into 738b6882995b488980087e0aa362d34a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:03,445 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c122f231a59b575ec7f40dae2c00f4/A of 94c122f231a59b575ec7f40dae2c00f4 into 52e6406ba447492387bd87b1c0ac5340(size=31.7 K), total size for store is 31.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:03,445 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:53:03,445 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c122f231a59b575ec7f40dae2c00f4: 2024-11-20T16:53:03,445 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/A, priority=13, startTime=1732121583309; duration=0sec 2024-11-20T16:53:03,445 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., storeName=94c122f231a59b575ec7f40dae2c00f4/C, priority=12, startTime=1732121583311; duration=0sec 2024-11-20T16:53:03,445 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:03,445 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:C 2024-11-20T16:53:03,445 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:03,445 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c122f231a59b575ec7f40dae2c00f4:A 2024-11-20T16:53:04,673 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T16:53:10,027 INFO [Thread-670 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-670. 2024-11-20T16:53:10,027 DEBUG [Thread-670 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:10,027 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:10,027 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:10,027 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:10,087 INFO [Thread-674 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-674. 2024-11-20T16:53:10,088 DEBUG [Thread-674 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:10,088 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:10,088 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:10,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:10,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T16:53:10,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-20T16:53:10,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 127 2024-11-20T16:53:10,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-20T16:53:10,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 146 2024-11-20T16:53:10,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 129 2024-11-20T16:53:10,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T16:53:10,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2291 2024-11-20T16:53:10,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2271 2024-11-20T16:53:10,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T16:53:10,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1022 2024-11-20T16:53:10,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3064 rows 2024-11-20T16:53:10,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1035 2024-11-20T16:53:10,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3098 rows 2024-11-20T16:53:10,088 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T16:53:10,088 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T16:53:10,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:10,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:10,089 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:10,089 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T16:53:10,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T16:53:10,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T16:53:10,092 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121590092"}]},"ts":"1732121590092"} 2024-11-20T16:53:10,094 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T16:53:10,094 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T16:53:10,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T16:53:10,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c122f231a59b575ec7f40dae2c00f4, UNASSIGN}] 2024-11-20T16:53:10,096 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c122f231a59b575ec7f40dae2c00f4, UNASSIGN 2024-11-20T16:53:10,097 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=94c122f231a59b575ec7f40dae2c00f4, regionState=CLOSING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:10,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c122f231a59b575ec7f40dae2c00f4, UNASSIGN because future has completed 2024-11-20T16:53:10,098 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T16:53:10,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:53:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T16:53:10,251 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:10,251 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T16:53:10,251 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing 94c122f231a59b575ec7f40dae2c00f4, disabling compactions & flushes 2024-11-20T16:53:10,251 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:53:10,251 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:53:10,251 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. after waiting 0 ms 2024-11-20T16:53:10,251 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:53:10,251 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing 94c122f231a59b575ec7f40dae2c00f4 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-20T16:53:10,252 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=A 2024-11-20T16:53:10,252 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:10,252 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=B 2024-11-20T16:53:10,252 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:10,252 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94c122f231a59b575ec7f40dae2c00f4, store=C 2024-11-20T16:53:10,252 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:10,259 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205b0f70bfe3b1490c80119966b73d25b3_94c122f231a59b575ec7f40dae2c00f4 is 50, key is test_row_2/A:col10/1732121590086/Put/seqid=0 2024-11-20T16:53:10,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742099_1275 (size=7374) 2024-11-20T16:53:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T16:53:10,663 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:10,667 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205b0f70bfe3b1490c80119966b73d25b3_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205b0f70bfe3b1490c80119966b73d25b3_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:10,668 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/6b393d9632664410a5770220e692b2b8, store: [table=TestAcidGuarantees family=A region=94c122f231a59b575ec7f40dae2c00f4] 2024-11-20T16:53:10,669 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/6b393d9632664410a5770220e692b2b8 is 175, key is test_row_2/A:col10/1732121590086/Put/seqid=0 2024-11-20T16:53:10,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742100_1276 (size=13865) 2024-11-20T16:53:10,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T16:53:11,073 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=540, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/6b393d9632664410a5770220e692b2b8 2024-11-20T16:53:11,079 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/bf27a283d45343378e19fa65b1282bb2 is 50, key is test_row_2/B:col10/1732121590086/Put/seqid=0 2024-11-20T16:53:11,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742101_1277 (size=7415) 2024-11-20T16:53:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T16:53:11,483 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/bf27a283d45343378e19fa65b1282bb2 2024-11-20T16:53:11,489 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/2000efabf98244eb90025ccbababea7a is 50, key is test_row_2/C:col10/1732121590086/Put/seqid=0 2024-11-20T16:53:11,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742102_1278 (size=7415) 2024-11-20T16:53:11,892 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/2000efabf98244eb90025ccbababea7a 2024-11-20T16:53:11,897 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/A/6b393d9632664410a5770220e692b2b8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/6b393d9632664410a5770220e692b2b8 2024-11-20T16:53:11,900 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/6b393d9632664410a5770220e692b2b8, entries=50, sequenceid=540, filesize=13.5 K 2024-11-20T16:53:11,901 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/B/bf27a283d45343378e19fa65b1282bb2 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/bf27a283d45343378e19fa65b1282bb2 2024-11-20T16:53:11,904 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/bf27a283d45343378e19fa65b1282bb2, entries=50, sequenceid=540, filesize=7.2 K 2024-11-20T16:53:11,904 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/.tmp/C/2000efabf98244eb90025ccbababea7a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/2000efabf98244eb90025ccbababea7a 2024-11-20T16:53:11,907 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/2000efabf98244eb90025ccbababea7a, entries=50, sequenceid=540, filesize=7.2 K 2024-11-20T16:53:11,908 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 94c122f231a59b575ec7f40dae2c00f4 in 1657ms, sequenceid=540, compaction requested=false 2024-11-20T16:53:11,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/8765d4ce8d884ce4af94ba8e5561d865, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f00e8aad3f594453a622ba28c883ee27, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/c81730be4a5f41b99877ac8a7f5ad622, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/d7689b73f8b64156955240884e503df5, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/2cf44e08c23b42ab93bb51652adc8b8f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0b9fe730c513465ea746d13fe026c891, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b1034b44bce24503b6c10b063a49e75c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/ea0e27a4b39d45fa9f5da03cddb2ecd4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/c14edccfe7bd4e2592f6c5c3fc7d5478, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/7ce54154e9ac4f09a8b06ec37f1a536b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/bf8db4e66c094f888417ed064a3a9e94, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a4cae39517864a528fc714ca54690ccd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a7ed4166bb714da3adcd077b5a522a6a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b250a3a08ab6423fa16d95c4e88a5da7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0da8b82ab450495288e560ed059a5b24, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/5148a7260afd4f3da30af261eda52723, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/16d5c59f56ab4ed2ace4dbeebf783ce2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f6f29ae178b64c548f50ab3f9e3e0f29, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/4bf25b1d423e4aec8fb931d7866a807c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/500e6a0a0ba841cd8eecb0999f62fad7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/260649ca8cac4860968ff6e284de539e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/e664616ab43f4d4397c829a1ad111247, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/e21ae777ddfb4f6d81c3666ba76c4293, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0f6d161b903c42ac8347a524a09fabbe, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/73eeda4307f7426092e70b384c944334, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/837ffa3bb1f34e2e937b20435c61b705, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/9748399cec2740d78426ff0b4245ef24, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0b22cbc42c67425bbb5ba4de926cfc8d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/2e186cbacfe242aa83673342bd54951d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/9f94e3775a0048a5a675cacfd2f5d4fc, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b4ecf440859c4f3089768802a9f457ef, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f9ed673fae46490bbb69dc0f8c33f9fc, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/ac8149d44e704e4293364e969904c85c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a45c59ba962242df919db257a10e4d2d] to archive 2024-11-20T16:53:11,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T16:53:11,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/8765d4ce8d884ce4af94ba8e5561d865 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/8765d4ce8d884ce4af94ba8e5561d865 2024-11-20T16:53:11,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f00e8aad3f594453a622ba28c883ee27 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f00e8aad3f594453a622ba28c883ee27 2024-11-20T16:53:11,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/c81730be4a5f41b99877ac8a7f5ad622 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/c81730be4a5f41b99877ac8a7f5ad622 2024-11-20T16:53:11,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/d7689b73f8b64156955240884e503df5 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/d7689b73f8b64156955240884e503df5 2024-11-20T16:53:11,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/2cf44e08c23b42ab93bb51652adc8b8f to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/2cf44e08c23b42ab93bb51652adc8b8f 2024-11-20T16:53:11,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0b9fe730c513465ea746d13fe026c891 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0b9fe730c513465ea746d13fe026c891 2024-11-20T16:53:11,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b1034b44bce24503b6c10b063a49e75c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b1034b44bce24503b6c10b063a49e75c 2024-11-20T16:53:11,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/ea0e27a4b39d45fa9f5da03cddb2ecd4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/ea0e27a4b39d45fa9f5da03cddb2ecd4 2024-11-20T16:53:11,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/c14edccfe7bd4e2592f6c5c3fc7d5478 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/c14edccfe7bd4e2592f6c5c3fc7d5478 2024-11-20T16:53:11,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/7ce54154e9ac4f09a8b06ec37f1a536b to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/7ce54154e9ac4f09a8b06ec37f1a536b 2024-11-20T16:53:11,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/bf8db4e66c094f888417ed064a3a9e94 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/bf8db4e66c094f888417ed064a3a9e94 2024-11-20T16:53:11,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a4cae39517864a528fc714ca54690ccd to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a4cae39517864a528fc714ca54690ccd 2024-11-20T16:53:11,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a7ed4166bb714da3adcd077b5a522a6a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a7ed4166bb714da3adcd077b5a522a6a 2024-11-20T16:53:11,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b250a3a08ab6423fa16d95c4e88a5da7 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b250a3a08ab6423fa16d95c4e88a5da7 2024-11-20T16:53:11,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0da8b82ab450495288e560ed059a5b24 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0da8b82ab450495288e560ed059a5b24 2024-11-20T16:53:11,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/5148a7260afd4f3da30af261eda52723 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/5148a7260afd4f3da30af261eda52723 2024-11-20T16:53:11,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/16d5c59f56ab4ed2ace4dbeebf783ce2 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/16d5c59f56ab4ed2ace4dbeebf783ce2 2024-11-20T16:53:11,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f6f29ae178b64c548f50ab3f9e3e0f29 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f6f29ae178b64c548f50ab3f9e3e0f29 2024-11-20T16:53:11,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/4bf25b1d423e4aec8fb931d7866a807c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/4bf25b1d423e4aec8fb931d7866a807c 2024-11-20T16:53:11,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/500e6a0a0ba841cd8eecb0999f62fad7 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/500e6a0a0ba841cd8eecb0999f62fad7 2024-11-20T16:53:11,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/260649ca8cac4860968ff6e284de539e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/260649ca8cac4860968ff6e284de539e 2024-11-20T16:53:11,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/e664616ab43f4d4397c829a1ad111247 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/e664616ab43f4d4397c829a1ad111247 2024-11-20T16:53:11,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/e21ae777ddfb4f6d81c3666ba76c4293 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/e21ae777ddfb4f6d81c3666ba76c4293 2024-11-20T16:53:11,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0f6d161b903c42ac8347a524a09fabbe to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0f6d161b903c42ac8347a524a09fabbe 2024-11-20T16:53:11,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/73eeda4307f7426092e70b384c944334 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/73eeda4307f7426092e70b384c944334 2024-11-20T16:53:11,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/837ffa3bb1f34e2e937b20435c61b705 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/837ffa3bb1f34e2e937b20435c61b705 2024-11-20T16:53:11,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/9748399cec2740d78426ff0b4245ef24 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/9748399cec2740d78426ff0b4245ef24 2024-11-20T16:53:11,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0b22cbc42c67425bbb5ba4de926cfc8d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/0b22cbc42c67425bbb5ba4de926cfc8d 2024-11-20T16:53:11,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/2e186cbacfe242aa83673342bd54951d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/2e186cbacfe242aa83673342bd54951d 2024-11-20T16:53:11,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/9f94e3775a0048a5a675cacfd2f5d4fc to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/9f94e3775a0048a5a675cacfd2f5d4fc 2024-11-20T16:53:11,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b4ecf440859c4f3089768802a9f457ef to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/b4ecf440859c4f3089768802a9f457ef 2024-11-20T16:53:11,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f9ed673fae46490bbb69dc0f8c33f9fc to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/f9ed673fae46490bbb69dc0f8c33f9fc 2024-11-20T16:53:11,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/ac8149d44e704e4293364e969904c85c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/ac8149d44e704e4293364e969904c85c 2024-11-20T16:53:11,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a45c59ba962242df919db257a10e4d2d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/a45c59ba962242df919db257a10e4d2d 2024-11-20T16:53:11,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/cd4d17870ea64306967e2966a9e55cae, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/f465769f091b44e6865641ea416beed7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/94f2bd42818e4b89b48d1ecea55c5c6c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/d5ad19dd112a430cb1ac3695bf2907df, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/cd25ff59abb54b74b51eb23ae5459ebb, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/3fd67cbf28c249d4a19291840a381364, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/f7369e6561ef45cf90fbdc773d90cbb1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/e9a3e467f30744a4a1e0a1a665ebb94f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/a0d629637c4343d3a17583e53397cfd3, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/4f1da40bdfb241328ee3e073fef3991f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/6c81ffed432544f9ba36b02a15452b82, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/00f226bc322d446ab6c56aae80c1a63f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/a95616ac925d4c54b5351153ba78bec0, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/9ca2ae43eec4416cb37d32e7994b416c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/2f4c5a13c206403e90e1457975a3f24c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/8592cb3959a4430bbe0cac2bb5da331f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/db74a94fb9ff41fda19afafc30f2ae06, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/b858dd0e136d4acf97e77caa24dfc799, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/45424478ab5045f6be398ac9d377bafa, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/309a76f4fe454d2fa0dc7ad57615ba1a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/e0e7f79d4d0447bd9a4e975ebf95efb2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/5df02ce79aa1435b8f0d1afad8075a5e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/face194c131b49309621b70e07375108, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/80bd8bdba6fc4b72b305c945f6a1e744, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/7fe13f3adf4e4d1da71a12f02f31ada2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/e54431d39318493d8d93be0f31a6e288, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/457c79ae0105462ca860925b4e0d6738, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/12e6ddfd0f9049a894d86235bb553988, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/0d31c520bda342a19b3cb006db310591, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/51d7bad1b8a04a69b5a4b7aba99d5bda, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/0a666df3b4104d7f81c8f476605e6dae, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/d3cf9ba777c3483da862ae7875a846ba, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/293ea847926f4ccb8c031a6158644a3f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/c630282f8b584546b2040f4545b5c12e] to archive 2024-11-20T16:53:11,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T16:53:11,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/cd4d17870ea64306967e2966a9e55cae to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/cd4d17870ea64306967e2966a9e55cae 2024-11-20T16:53:11,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/f465769f091b44e6865641ea416beed7 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/f465769f091b44e6865641ea416beed7 2024-11-20T16:53:11,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/94f2bd42818e4b89b48d1ecea55c5c6c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/94f2bd42818e4b89b48d1ecea55c5c6c 2024-11-20T16:53:11,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/d5ad19dd112a430cb1ac3695bf2907df to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/d5ad19dd112a430cb1ac3695bf2907df 2024-11-20T16:53:11,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/cd25ff59abb54b74b51eb23ae5459ebb to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/cd25ff59abb54b74b51eb23ae5459ebb 2024-11-20T16:53:11,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/3fd67cbf28c249d4a19291840a381364 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/3fd67cbf28c249d4a19291840a381364 2024-11-20T16:53:11,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/f7369e6561ef45cf90fbdc773d90cbb1 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/f7369e6561ef45cf90fbdc773d90cbb1 2024-11-20T16:53:11,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/e9a3e467f30744a4a1e0a1a665ebb94f to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/e9a3e467f30744a4a1e0a1a665ebb94f 2024-11-20T16:53:11,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/a0d629637c4343d3a17583e53397cfd3 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/a0d629637c4343d3a17583e53397cfd3 2024-11-20T16:53:11,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/4f1da40bdfb241328ee3e073fef3991f to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/4f1da40bdfb241328ee3e073fef3991f 2024-11-20T16:53:11,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/6c81ffed432544f9ba36b02a15452b82 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/6c81ffed432544f9ba36b02a15452b82 2024-11-20T16:53:11,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/00f226bc322d446ab6c56aae80c1a63f to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/00f226bc322d446ab6c56aae80c1a63f 2024-11-20T16:53:11,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/a95616ac925d4c54b5351153ba78bec0 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/a95616ac925d4c54b5351153ba78bec0 2024-11-20T16:53:11,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/9ca2ae43eec4416cb37d32e7994b416c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/9ca2ae43eec4416cb37d32e7994b416c 2024-11-20T16:53:11,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/2f4c5a13c206403e90e1457975a3f24c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/2f4c5a13c206403e90e1457975a3f24c 2024-11-20T16:53:11,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/8592cb3959a4430bbe0cac2bb5da331f to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/8592cb3959a4430bbe0cac2bb5da331f 2024-11-20T16:53:11,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/db74a94fb9ff41fda19afafc30f2ae06 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/db74a94fb9ff41fda19afafc30f2ae06 2024-11-20T16:53:11,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/b858dd0e136d4acf97e77caa24dfc799 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/b858dd0e136d4acf97e77caa24dfc799 2024-11-20T16:53:11,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/45424478ab5045f6be398ac9d377bafa to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/45424478ab5045f6be398ac9d377bafa 2024-11-20T16:53:11,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/309a76f4fe454d2fa0dc7ad57615ba1a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/309a76f4fe454d2fa0dc7ad57615ba1a 2024-11-20T16:53:11,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/e0e7f79d4d0447bd9a4e975ebf95efb2 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/e0e7f79d4d0447bd9a4e975ebf95efb2 2024-11-20T16:53:11,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/5df02ce79aa1435b8f0d1afad8075a5e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/5df02ce79aa1435b8f0d1afad8075a5e 2024-11-20T16:53:11,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/face194c131b49309621b70e07375108 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/face194c131b49309621b70e07375108 2024-11-20T16:53:11,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/80bd8bdba6fc4b72b305c945f6a1e744 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/80bd8bdba6fc4b72b305c945f6a1e744 2024-11-20T16:53:11,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/7fe13f3adf4e4d1da71a12f02f31ada2 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/7fe13f3adf4e4d1da71a12f02f31ada2 2024-11-20T16:53:11,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/e54431d39318493d8d93be0f31a6e288 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/e54431d39318493d8d93be0f31a6e288 2024-11-20T16:53:11,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/457c79ae0105462ca860925b4e0d6738 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/457c79ae0105462ca860925b4e0d6738 2024-11-20T16:53:11,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/12e6ddfd0f9049a894d86235bb553988 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/12e6ddfd0f9049a894d86235bb553988 2024-11-20T16:53:11,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/0d31c520bda342a19b3cb006db310591 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/0d31c520bda342a19b3cb006db310591 2024-11-20T16:53:11,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/51d7bad1b8a04a69b5a4b7aba99d5bda to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/51d7bad1b8a04a69b5a4b7aba99d5bda 2024-11-20T16:53:11,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/0a666df3b4104d7f81c8f476605e6dae to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/0a666df3b4104d7f81c8f476605e6dae 2024-11-20T16:53:11,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/d3cf9ba777c3483da862ae7875a846ba to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/d3cf9ba777c3483da862ae7875a846ba 2024-11-20T16:53:11,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/293ea847926f4ccb8c031a6158644a3f to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/293ea847926f4ccb8c031a6158644a3f 2024-11-20T16:53:11,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/c630282f8b584546b2040f4545b5c12e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/c630282f8b584546b2040f4545b5c12e 2024-11-20T16:53:11,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/cd2cc91b51e247ebbf2b725957e2eb7c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/dda406cb62594dff9d41e7b4dffbe34d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/8e737d7a89ed467d86382c1af0801e0a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/3058e11aa4b4441d822d46e501fc538e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/6df5e8b2a8964392a2678e129016f77e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/aa834389a15d4822bf6fbaa2bc93d800, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/d30aa1da9d4b4163b96770292fb7ffc4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/dfebf2bcf86d40c9a600657244d63e27, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/e700478c5ce749febfcd4a126e46af0e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/56a7c40ae8ac4ef0a721852a946ba1e9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/56fd725fcbdf43d38867f9f9d9994db8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/d46f5a97021a481cab6a1dcebb5b197a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/9f2430e4fb2e484395c42af887648055, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/6d16fa124b044b39a04301d622e1e8de, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/609d9a1e52ae48629924d0728c393359, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/67deb19345f2400bae46b02f944d04b3, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/2d607505a10142eebcbb2710faa7cf6d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/dd2af85dca58427093895ef0d223393c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/8ca8f765bb2945ad88d38b34d455233b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/bea8a3434f884f31b15a936ce6984500, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/bb37cf09a16a4f15896ac50f8abd1908, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/cf3d7e7991c64e2e8a57b469c08b53db, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/9d25a319e03f405496c3e543c15de5f8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/64b8fa1df23440df9f0abd24e8f42c52, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/0671bcd4fa0546ad9f820aa426efbd1a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/0a69267ca41f4a179baca16de3d94a98, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/a26922420d404a7abf34fb03e4f944ac, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/26aa585f23b846c5a9077c6f4f855e9d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/823236e23a2042b098b5294ed9865ba8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/1a38f9c4375f4cac983880b5f08faf87, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/a3ed876167d8465c9a399441a3f59fdc, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/3598bb7bbf5c430da2e2e54e26c01f9c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/f59b4946f8544778a1cfcf3bfceb7e2a] to archive 2024-11-20T16:53:11,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T16:53:11,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/cd2cc91b51e247ebbf2b725957e2eb7c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/cd2cc91b51e247ebbf2b725957e2eb7c 2024-11-20T16:53:11,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/dda406cb62594dff9d41e7b4dffbe34d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/dda406cb62594dff9d41e7b4dffbe34d 2024-11-20T16:53:11,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/8e737d7a89ed467d86382c1af0801e0a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/8e737d7a89ed467d86382c1af0801e0a 2024-11-20T16:53:12,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/3058e11aa4b4441d822d46e501fc538e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/3058e11aa4b4441d822d46e501fc538e 2024-11-20T16:53:12,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/6df5e8b2a8964392a2678e129016f77e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/6df5e8b2a8964392a2678e129016f77e 2024-11-20T16:53:12,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/aa834389a15d4822bf6fbaa2bc93d800 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/aa834389a15d4822bf6fbaa2bc93d800 2024-11-20T16:53:12,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/d30aa1da9d4b4163b96770292fb7ffc4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/d30aa1da9d4b4163b96770292fb7ffc4 2024-11-20T16:53:12,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/dfebf2bcf86d40c9a600657244d63e27 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/dfebf2bcf86d40c9a600657244d63e27 2024-11-20T16:53:12,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/e700478c5ce749febfcd4a126e46af0e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/e700478c5ce749febfcd4a126e46af0e 2024-11-20T16:53:12,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/56a7c40ae8ac4ef0a721852a946ba1e9 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/56a7c40ae8ac4ef0a721852a946ba1e9 2024-11-20T16:53:12,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/56fd725fcbdf43d38867f9f9d9994db8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/56fd725fcbdf43d38867f9f9d9994db8 2024-11-20T16:53:12,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/d46f5a97021a481cab6a1dcebb5b197a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/d46f5a97021a481cab6a1dcebb5b197a 2024-11-20T16:53:12,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/9f2430e4fb2e484395c42af887648055 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/9f2430e4fb2e484395c42af887648055 2024-11-20T16:53:12,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/6d16fa124b044b39a04301d622e1e8de to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/6d16fa124b044b39a04301d622e1e8de 2024-11-20T16:53:12,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/609d9a1e52ae48629924d0728c393359 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/609d9a1e52ae48629924d0728c393359 2024-11-20T16:53:12,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/67deb19345f2400bae46b02f944d04b3 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/67deb19345f2400bae46b02f944d04b3 2024-11-20T16:53:12,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/2d607505a10142eebcbb2710faa7cf6d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/2d607505a10142eebcbb2710faa7cf6d 2024-11-20T16:53:12,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/dd2af85dca58427093895ef0d223393c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/dd2af85dca58427093895ef0d223393c 2024-11-20T16:53:12,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/8ca8f765bb2945ad88d38b34d455233b to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/8ca8f765bb2945ad88d38b34d455233b 2024-11-20T16:53:12,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/bea8a3434f884f31b15a936ce6984500 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/bea8a3434f884f31b15a936ce6984500 2024-11-20T16:53:12,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/bb37cf09a16a4f15896ac50f8abd1908 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/bb37cf09a16a4f15896ac50f8abd1908 2024-11-20T16:53:12,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/cf3d7e7991c64e2e8a57b469c08b53db to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/cf3d7e7991c64e2e8a57b469c08b53db 2024-11-20T16:53:12,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/9d25a319e03f405496c3e543c15de5f8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/9d25a319e03f405496c3e543c15de5f8 2024-11-20T16:53:12,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/64b8fa1df23440df9f0abd24e8f42c52 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/64b8fa1df23440df9f0abd24e8f42c52 2024-11-20T16:53:12,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/0671bcd4fa0546ad9f820aa426efbd1a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/0671bcd4fa0546ad9f820aa426efbd1a 2024-11-20T16:53:12,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/0a69267ca41f4a179baca16de3d94a98 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/0a69267ca41f4a179baca16de3d94a98 2024-11-20T16:53:12,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/a26922420d404a7abf34fb03e4f944ac to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/a26922420d404a7abf34fb03e4f944ac 2024-11-20T16:53:12,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/26aa585f23b846c5a9077c6f4f855e9d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/26aa585f23b846c5a9077c6f4f855e9d 2024-11-20T16:53:12,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/823236e23a2042b098b5294ed9865ba8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/823236e23a2042b098b5294ed9865ba8 2024-11-20T16:53:12,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/1a38f9c4375f4cac983880b5f08faf87 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/1a38f9c4375f4cac983880b5f08faf87 2024-11-20T16:53:12,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/a3ed876167d8465c9a399441a3f59fdc to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/a3ed876167d8465c9a399441a3f59fdc 2024-11-20T16:53:12,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/3598bb7bbf5c430da2e2e54e26c01f9c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/3598bb7bbf5c430da2e2e54e26c01f9c 2024-11-20T16:53:12,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/f59b4946f8544778a1cfcf3bfceb7e2a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/f59b4946f8544778a1cfcf3bfceb7e2a 2024-11-20T16:53:12,043 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/recovered.edits/543.seqid, newMaxSeqId=543, maxSeqId=4 2024-11-20T16:53:12,044 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. 2024-11-20T16:53:12,044 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for 94c122f231a59b575ec7f40dae2c00f4: Waiting for close lock at 1732121590251Running coprocessor pre-close hooks at 1732121590251Disabling compacts and flushes for region at 1732121590251Disabling writes for close at 1732121590251Obtaining lock to block concurrent updates at 1732121590251Preparing flush snapshotting stores in 94c122f231a59b575ec7f40dae2c00f4 at 1732121590251Finished memstore snapshotting TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732121590252 (+1 ms)Flushing stores of TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4. at 1732121590252Flushing 94c122f231a59b575ec7f40dae2c00f4/A: creating writer at 1732121590252Flushing mob file 94c122f231a59b575ec7f40dae2c00f4/A: appending metadata at 1732121590259 (+7 ms)Flushing mob file 94c122f231a59b575ec7f40dae2c00f4/A: closing flushed file at 1732121590259Flushing 94c122f231a59b575ec7f40dae2c00f4/A: appending metadata at 1732121590668 (+409 ms)Flushing 94c122f231a59b575ec7f40dae2c00f4/A: closing flushed file at 1732121590668Flushing 94c122f231a59b575ec7f40dae2c00f4/B: creating writer at 1732121591076 (+408 ms)Flushing 94c122f231a59b575ec7f40dae2c00f4/B: appending metadata at 1732121591079 (+3 ms)Flushing 94c122f231a59b575ec7f40dae2c00f4/B: closing flushed file at 1732121591079Flushing 94c122f231a59b575ec7f40dae2c00f4/C: creating writer at 1732121591486 (+407 ms)Flushing 94c122f231a59b575ec7f40dae2c00f4/C: appending metadata at 1732121591488 (+2 ms)Flushing 94c122f231a59b575ec7f40dae2c00f4/C: closing flushed file at 1732121591488Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77173a88: reopening flushed file at 1732121591896 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72c6183e: reopening flushed file at 1732121591900 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c0e7aed: reopening flushed file at 1732121591904 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 94c122f231a59b575ec7f40dae2c00f4 in 1657ms, sequenceid=540, compaction requested=false at 1732121591908 (+4 ms)Writing region close event to WAL at 1732121592039 (+131 ms)Running coprocessor post-close hooks at 1732121592044 (+5 ms)Closed at 1732121592044 2024-11-20T16:53:12,046 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed 94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,046 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=94c122f231a59b575ec7f40dae2c00f4, regionState=CLOSED 2024-11-20T16:53:12,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 because future has completed 2024-11-20T16:53:12,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-20T16:53:12,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure 94c122f231a59b575ec7f40dae2c00f4, server=b60eac9bda43,33929,1732121526384 in 1.9510 sec 2024-11-20T16:53:12,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-20T16:53:12,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c122f231a59b575ec7f40dae2c00f4, UNASSIGN in 1.9540 sec 2024-11-20T16:53:12,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-20T16:53:12,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9580 sec 2024-11-20T16:53:12,055 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121592055"}]},"ts":"1732121592055"} 2024-11-20T16:53:12,056 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T16:53:12,056 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T16:53:12,058 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9680 sec 2024-11-20T16:53:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T16:53:12,236 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:12,237 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T16:53:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:53:12,239 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:53:12,239 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=66, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:53:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T16:53:12,242 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,245 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A, FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B, FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C, FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/recovered.edits] 2024-11-20T16:53:12,247 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/52e6406ba447492387bd87b1c0ac5340 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/52e6406ba447492387bd87b1c0ac5340 2024-11-20T16:53:12,248 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/6b393d9632664410a5770220e692b2b8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/A/6b393d9632664410a5770220e692b2b8 2024-11-20T16:53:12,251 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/6bb9b2409d084f30ad8fff9b320564ee to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/6bb9b2409d084f30ad8fff9b320564ee 2024-11-20T16:53:12,255 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/bf27a283d45343378e19fa65b1282bb2 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/B/bf27a283d45343378e19fa65b1282bb2 2024-11-20T16:53:12,257 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/2000efabf98244eb90025ccbababea7a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/2000efabf98244eb90025ccbababea7a 2024-11-20T16:53:12,259 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/738b6882995b488980087e0aa362d34a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/C/738b6882995b488980087e0aa362d34a 2024-11-20T16:53:12,263 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/recovered.edits/543.seqid to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4/recovered.edits/543.seqid 2024-11-20T16:53:12,263 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,263 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T16:53:12,264 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T16:53:12,265 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T16:53:12,270 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200bbc3423cc1a469681c5952366c01155_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200bbc3423cc1a469681c5952366c01155_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,272 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201585d6a800af4a88b1fa1c86ca857a28_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201585d6a800af4a88b1fa1c86ca857a28_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,274 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201b09407288a841d8ad664cf9d1d721a7_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201b09407288a841d8ad664cf9d1d721a7_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,275 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112046351b3fc3d2475488ca304aee5eeff1_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112046351b3fc3d2475488ca304aee5eeff1_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,277 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112046837b5abe8646c59f542f506e8e72bb_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112046837b5abe8646c59f542f506e8e72bb_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,278 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204976c822146e4eb19019b454a9ac27ac_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204976c822146e4eb19019b454a9ac27ac_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,279 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204be369d75ce04b5688135d24eb224ef4_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204be369d75ce04b5688135d24eb224ef4_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,280 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120568972535e0b47dfb5651f7212e66940_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120568972535e0b47dfb5651f7212e66940_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,281 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205b0f70bfe3b1490c80119966b73d25b3_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205b0f70bfe3b1490c80119966b73d25b3_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,282 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206a09a8ea90814c5db194c17ae90d0412_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206a09a8ea90814c5db194c17ae90d0412_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,284 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206c3c3442187f4451aec25bfa2b347c59_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206c3c3442187f4451aec25bfa2b347c59_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,285 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206c71c919d46a4ef3bdc7b0e34fe9466e_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206c71c919d46a4ef3bdc7b0e34fe9466e_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,286 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112088010b2b323f47cd8d275a9b0fe1fe2d_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112088010b2b323f47cd8d275a9b0fe1fe2d_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,287 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208eff79d3b765418c954795db590f5438_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208eff79d3b765418c954795db590f5438_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,288 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209aac1ba1c3194e61a9fcaecc3b2c4fee_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209aac1ba1c3194e61a9fcaecc3b2c4fee_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,289 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a01e5e5ba3cf4d53b68e506139eef1b5_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a01e5e5ba3cf4d53b68e506139eef1b5_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,290 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a7a644824d5d440b9fcf7d21630c2674_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a7a644824d5d440b9fcf7d21630c2674_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,290 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b505bdbbd5824921be03f59d5a5c41ce_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b505bdbbd5824921be03f59d5a5c41ce_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,291 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b9fdc3af8dce4715914006178acf1964_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b9fdc3af8dce4715914006178acf1964_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,292 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120be5f9f2a916a490db1a7cbf21aca0c49_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120be5f9f2a916a490db1a7cbf21aca0c49_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,293 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c41b6dc77e1c487aa86d2556f1f5d303_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c41b6dc77e1c487aa86d2556f1f5d303_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,294 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d1ba5828cea244f4a15be52831dd59c8_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d1ba5828cea244f4a15be52831dd59c8_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,296 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d3e6ce7e7cd34142a40ad9ebc1373bc5_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d3e6ce7e7cd34142a40ad9ebc1373bc5_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,297 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e7e77fe4c94f46f7830ab8464f9d9ea0_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e7e77fe4c94f46f7830ab8464f9d9ea0_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,300 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e8064c236c504326a74743fd8488cf88_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e8064c236c504326a74743fd8488cf88_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,304 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f38598172cf2437ebe652a2047930f7c_94c122f231a59b575ec7f40dae2c00f4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f38598172cf2437ebe652a2047930f7c_94c122f231a59b575ec7f40dae2c00f4 2024-11-20T16:53:12,305 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T16:53:12,307 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=66, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:53:12,310 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T16:53:12,312 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T16:53:12,313 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=66, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:53:12,313 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T16:53:12,313 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732121592313"}]},"ts":"9223372036854775807"} 2024-11-20T16:53:12,315 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T16:53:12,315 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 94c122f231a59b575ec7f40dae2c00f4, NAME => 'TestAcidGuarantees,,1732121559024.94c122f231a59b575ec7f40dae2c00f4.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T16:53:12,315 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T16:53:12,315 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732121592315"}]},"ts":"9223372036854775807"} 2024-11-20T16:53:12,317 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T16:53:12,318 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=66, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:53:12,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 81 msec 2024-11-20T16:53:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T16:53:12,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T16:53:12,346 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:12,356 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 248), OpenFileDescriptor=455 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=580 (was 402) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6015 (was 6603) 2024-11-20T16:53:12,371 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=580, ProcessCount=11, AvailableMemoryMB=6018 2024-11-20T16:53:12,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T16:53:12,373 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T16:53:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T16:53:12,375 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T16:53:12,375 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:12,375 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 67 2024-11-20T16:53:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-20T16:53:12,380 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T16:53:12,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742103_1279 (size=1061) 2024-11-20T16:53:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-20T16:53:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-20T16:53:12,799 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625 2024-11-20T16:53:12,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742104_1280 (size=53) 2024-11-20T16:53:13,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-20T16:53:13,204 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:53:13,204 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e1410d95b5f65cde4303e83d1cb47b81, disabling compactions & flushes 2024-11-20T16:53:13,204 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:13,204 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:13,204 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. after waiting 0 ms 2024-11-20T16:53:13,204 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:13,204 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:13,204 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e1410d95b5f65cde4303e83d1cb47b81: Waiting for close lock at 1732121593204Disabling compacts and flushes for region at 1732121593204Disabling writes for close at 1732121593204Writing region close event to WAL at 1732121593204Closed at 1732121593204 2024-11-20T16:53:13,205 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T16:53:13,205 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732121593205"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732121593205"}]},"ts":"1732121593205"} 2024-11-20T16:53:13,207 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T16:53:13,208 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T16:53:13,208 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121593208"}]},"ts":"1732121593208"} 2024-11-20T16:53:13,209 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T16:53:13,210 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1410d95b5f65cde4303e83d1cb47b81, ASSIGN}] 2024-11-20T16:53:13,210 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1410d95b5f65cde4303e83d1cb47b81, ASSIGN 2024-11-20T16:53:13,211 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1410d95b5f65cde4303e83d1cb47b81, ASSIGN; state=OFFLINE, location=b60eac9bda43,33929,1732121526384; forceNewPlan=false, retain=false 2024-11-20T16:53:13,362 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=e1410d95b5f65cde4303e83d1cb47b81, regionState=OPENING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:13,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1410d95b5f65cde4303e83d1cb47b81, ASSIGN because future has completed 2024-11-20T16:53:13,363 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:53:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-20T16:53:13,518 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:13,518 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7752): Opening region: {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} 2024-11-20T16:53:13,518 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:13,518 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:53:13,518 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7794): checking encryption for e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:13,518 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7797): checking classloading for e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:13,519 INFO [StoreOpener-e1410d95b5f65cde4303e83d1cb47b81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:13,520 INFO [StoreOpener-e1410d95b5f65cde4303e83d1cb47b81-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:53:13,521 INFO [StoreOpener-e1410d95b5f65cde4303e83d1cb47b81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1410d95b5f65cde4303e83d1cb47b81 columnFamilyName A 2024-11-20T16:53:13,521 DEBUG [StoreOpener-e1410d95b5f65cde4303e83d1cb47b81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:13,521 INFO [StoreOpener-e1410d95b5f65cde4303e83d1cb47b81-1 {}] regionserver.HStore(327): Store=e1410d95b5f65cde4303e83d1cb47b81/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:53:13,521 INFO [StoreOpener-e1410d95b5f65cde4303e83d1cb47b81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:13,522 INFO [StoreOpener-e1410d95b5f65cde4303e83d1cb47b81-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:53:13,522 INFO [StoreOpener-e1410d95b5f65cde4303e83d1cb47b81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1410d95b5f65cde4303e83d1cb47b81 columnFamilyName B 2024-11-20T16:53:13,522 DEBUG [StoreOpener-e1410d95b5f65cde4303e83d1cb47b81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:13,523 INFO [StoreOpener-e1410d95b5f65cde4303e83d1cb47b81-1 {}] regionserver.HStore(327): Store=e1410d95b5f65cde4303e83d1cb47b81/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:53:13,523 INFO [StoreOpener-e1410d95b5f65cde4303e83d1cb47b81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:13,524 INFO [StoreOpener-e1410d95b5f65cde4303e83d1cb47b81-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:53:13,524 INFO [StoreOpener-e1410d95b5f65cde4303e83d1cb47b81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1410d95b5f65cde4303e83d1cb47b81 columnFamilyName C 2024-11-20T16:53:13,524 DEBUG [StoreOpener-e1410d95b5f65cde4303e83d1cb47b81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:13,524 INFO [StoreOpener-e1410d95b5f65cde4303e83d1cb47b81-1 {}] regionserver.HStore(327): Store=e1410d95b5f65cde4303e83d1cb47b81/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:53:13,525 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:13,525 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1038): replaying wal for e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:13,525 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:13,525 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:13,526 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1048): stopping wal replay for e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:13,526 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1060): Cleaning up temporary data for e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:13,527 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T16:53:13,527 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1093): writing seq id for e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:13,529 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T16:53:13,529 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1114): Opened e1410d95b5f65cde4303e83d1cb47b81; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68866130, jitterRate=0.026185303926467896}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T16:53:13,529 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:13,531 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1006): Region open journal for e1410d95b5f65cde4303e83d1cb47b81: Running coprocessor pre-open hook at 1732121593518Writing region info on filesystem at 1732121593518Initializing all the Stores at 1732121593519 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121593519Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121593519Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121593519Cleaning up temporary data from old regions at 1732121593526 (+7 ms)Running coprocessor post-open hooks at 1732121593529 (+3 ms)Region opened successfully at 1732121593531 (+2 ms) 2024-11-20T16:53:13,531 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., pid=69, masterSystemTime=1732121593515 2024-11-20T16:53:13,533 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:13,533 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:13,533 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=e1410d95b5f65cde4303e83d1cb47b81, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:13,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 because future has completed 2024-11-20T16:53:13,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-20T16:53:13,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 in 172 msec 2024-11-20T16:53:13,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-11-20T16:53:13,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1410d95b5f65cde4303e83d1cb47b81, ASSIGN in 328 msec 2024-11-20T16:53:13,539 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T16:53:13,540 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121593539"}]},"ts":"1732121593539"} 2024-11-20T16:53:13,541 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T16:53:13,542 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T16:53:13,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-20T16:53:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-20T16:53:14,527 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:14,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,528 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:14,528 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:14,530 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71f42948, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:14,532 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50f94bc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,533 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:14,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:14,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:14,536 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:14,536 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T16:53:14,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is b60eac9bda43,33377,1732121525625 2024-11-20T16:53:14,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3e0f0acc 2024-11-20T16:53:14,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T16:53:14,539 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T16:53:14,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:14,540 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:14,541 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5374e7f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:14,543 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c9dfd10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:14,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:14,545 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:14,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:14,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:14,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@81dacfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:14,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:14,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:14,547 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62c51b06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,548 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:14,548 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:14,549 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ce3ca99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:14,550 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:14,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a4868e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,552 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:14,552 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:14,553 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:14,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:14,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:14,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43a261e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:14,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:14,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:14,555 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:14,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@923ebcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:14,556 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:14,558 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64463f8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:14,559 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:14,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76405272, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:14,561 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:14,562 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:14,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:14,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:14,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2389ab81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:14,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:14,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:14,563 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:14,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c290b5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:14,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:14,565 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2df82a8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:14,567 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e933378, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:14,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:14,569 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:14,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:14,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:14,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260bcf6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:14,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:14,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:14,571 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63455081, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,572 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:14,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:14,573 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67eb2e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:14,575 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:14,575 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b851df3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:14,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:14,577 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:14,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:14,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:14,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a3b8ba6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:14,578 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:14,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@133683a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:14,580 DEBUG [Thread-1285 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:14,580 DEBUG [Thread-1281 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:14,580 DEBUG [Thread-1296 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:14,580 DEBUG [Thread-1290 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:14,581 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:14,581 DEBUG [Thread-1294 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:14,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:14,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:14,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:14,582 DEBUG [Thread-1288 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:14,582 DEBUG [Thread-1277 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:14,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-20T16:53:14,583 DEBUG [Thread-1277 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:14,583 DEBUG [Thread-1277 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:14,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T16:53:14,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:53:14,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:14,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:53:14,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:53:14,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:53:14,587 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:53:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:53:14,589 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:14,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:53:14,590 DEBUG [Thread-1292 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:14,592 DEBUG [Thread-1279 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:14,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:14,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:14,593 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:14,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:14,595 DEBUG [Thread-1283 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:14,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:14,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:53:14,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:53:14,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:14,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T16:53:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:53:14,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:14,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:14,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:14,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:14,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:14,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:14,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/fa2a0e8e70aa40a1a024d6eaf7079c5a is 50, key is test_row_0/A:col10/1732121594599/Put/seqid=0 2024-11-20T16:53:14,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742105_1281 (size=12001) 2024-11-20T16:53:14,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121594614, max timestamp=1732121594652Num uniques 859; 2024-11-20T16:53:14,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121594614, max timestamp=1732121594652Num uniques 859; 2024-11-20T16:53:14,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121594614, max timestamp=1732121594652Num uniques 859; 2024-11-20T16:53:14,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:53:14,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:53:14,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:53:14,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121654683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T16:53:14,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:14,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:14,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732121654685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121654686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:14,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:14,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45996 deadline: 1732121654688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:14,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:14,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:14,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:14,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121654695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:14,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:14,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T16:53:14,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:14,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:14,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:14,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:14,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:14,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732121654806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:14,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121654806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:14,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:14,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45996 deadline: 1732121654806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121654806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:14,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121654816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:14,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T16:53:14,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:14,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:14,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:14,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:14,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T16:53:15,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121655016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45996 deadline: 1732121655016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121655016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732121655016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121655026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/fa2a0e8e70aa40a1a024d6eaf7079c5a 2024-11-20T16:53:15,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T16:53:15,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:15,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:15,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:15,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:15,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:15,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/3d53b08e15824c7fa5495927a7015c45 is 50, key is test_row_0/B:col10/1732121594599/Put/seqid=0 2024-11-20T16:53:15,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742106_1282 (size=12001) 2024-11-20T16:53:15,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/3d53b08e15824c7fa5495927a7015c45 2024-11-20T16:53:15,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/03df61dfd79244448c1bdd4642cb145e is 50, key is test_row_0/C:col10/1732121594599/Put/seqid=0 2024-11-20T16:53:15,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T16:53:15,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:15,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:15,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:15,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:15,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T16:53:15,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742107_1283 (size=12001) 2024-11-20T16:53:15,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/03df61dfd79244448c1bdd4642cb145e 2024-11-20T16:53:15,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/fa2a0e8e70aa40a1a024d6eaf7079c5a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/fa2a0e8e70aa40a1a024d6eaf7079c5a 2024-11-20T16:53:15,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/fa2a0e8e70aa40a1a024d6eaf7079c5a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T16:53:15,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/3d53b08e15824c7fa5495927a7015c45 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/3d53b08e15824c7fa5495927a7015c45 2024-11-20T16:53:15,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/3d53b08e15824c7fa5495927a7015c45, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T16:53:15,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/03df61dfd79244448c1bdd4642cb145e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/03df61dfd79244448c1bdd4642cb145e 2024-11-20T16:53:15,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/03df61dfd79244448c1bdd4642cb145e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T16:53:15,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for e1410d95b5f65cde4303e83d1cb47b81 in 720ms, sequenceid=13, compaction requested=false 2024-11-20T16:53:15,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:15,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-20T16:53:15,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:15,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:15,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:15,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:15,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:15,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:15,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/7c6ab23c4b7943508516703f4181559e is 50, key is test_row_0/A:col10/1732121594684/Put/seqid=0 2024-11-20T16:53:15,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732121655349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121655352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45996 deadline: 1732121655354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121655355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T16:53:15,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:15,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:15,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:15,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:15,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:15,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121655363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742108_1284 (size=14341) 2024-11-20T16:53:15,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/7c6ab23c4b7943508516703f4181559e 2024-11-20T16:53:15,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/306ffc6c4e9c4ad28736484a0ea665e2 is 50, key is test_row_0/B:col10/1732121594684/Put/seqid=0 2024-11-20T16:53:15,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742109_1285 (size=12001) 2024-11-20T16:53:15,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/306ffc6c4e9c4ad28736484a0ea665e2 2024-11-20T16:53:15,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/b2bd8c3d505a4d50a2a005b64409607f is 50, key is test_row_0/C:col10/1732121594684/Put/seqid=0 2024-11-20T16:53:15,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121655471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121655471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45996 deadline: 1732121655471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732121655471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121655476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742110_1286 (size=12001) 2024-11-20T16:53:15,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T16:53:15,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:15,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:15,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:15,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:15,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:15,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T16:53:15,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:15,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:15,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:15,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:15,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:15,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45996 deadline: 1732121655687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121655686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121655687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732121655687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121655696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T16:53:15,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T16:53:15,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:15,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:15,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:15,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:15,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:15,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/b2bd8c3d505a4d50a2a005b64409607f 2024-11-20T16:53:15,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/7c6ab23c4b7943508516703f4181559e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/7c6ab23c4b7943508516703f4181559e 2024-11-20T16:53:15,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/7c6ab23c4b7943508516703f4181559e, entries=200, sequenceid=48, filesize=14.0 K 2024-11-20T16:53:15,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/306ffc6c4e9c4ad28736484a0ea665e2 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/306ffc6c4e9c4ad28736484a0ea665e2 2024-11-20T16:53:15,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/306ffc6c4e9c4ad28736484a0ea665e2, entries=150, sequenceid=48, filesize=11.7 K 2024-11-20T16:53:15,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/b2bd8c3d505a4d50a2a005b64409607f as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/b2bd8c3d505a4d50a2a005b64409607f 2024-11-20T16:53:15,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/b2bd8c3d505a4d50a2a005b64409607f, entries=150, sequenceid=48, filesize=11.7 K 2024-11-20T16:53:15,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for e1410d95b5f65cde4303e83d1cb47b81 in 613ms, sequenceid=48, compaction requested=false 2024-11-20T16:53:15,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:15,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T16:53:15,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:15,978 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T16:53:15,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:15,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:15,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:15,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:15,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:15,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:15,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/ce8c5af24dbb4a1294a695091f2b7e4a is 50, key is test_row_0/A:col10/1732121595346/Put/seqid=0 2024-11-20T16:53:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:16,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:16,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742111_1287 (size=12001) 2024-11-20T16:53:16,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121656020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732121596015, max timestamp=1732121596025Num uniques 859; 2024-11-20T16:53:16,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732121596015, max timestamp=1732121596025Num uniques 859; 2024-11-20T16:53:16,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732121596015, max timestamp=1732121596025Num uniques 859; 2024-11-20T16:53:16,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T16:53:16,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T16:53:16,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T16:53:16,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121656052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121656052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732121656053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45996 deadline: 1732121656060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121656137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121656166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732121656166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121656166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45996 deadline: 1732121656176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121656346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121656376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121656376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45996 deadline: 1732121656386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732121656386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,413 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/ce8c5af24dbb4a1294a695091f2b7e4a 2024-11-20T16:53:16,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/3b16e9aac3f0492da0738fcc4fe5f4ab is 50, key is test_row_0/B:col10/1732121595346/Put/seqid=0 2024-11-20T16:53:16,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742112_1288 (size=12001) 2024-11-20T16:53:16,462 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/3b16e9aac3f0492da0738fcc4fe5f4ab 2024-11-20T16:53:16,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/36cff06bde1f433db6fee33f67b3dc21 is 50, key is test_row_0/C:col10/1732121595346/Put/seqid=0 2024-11-20T16:53:16,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742113_1289 (size=12001) 2024-11-20T16:53:16,535 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/36cff06bde1f433db6fee33f67b3dc21 2024-11-20T16:53:16,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/ce8c5af24dbb4a1294a695091f2b7e4a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/ce8c5af24dbb4a1294a695091f2b7e4a 2024-11-20T16:53:16,554 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/ce8c5af24dbb4a1294a695091f2b7e4a, entries=150, sequenceid=59, filesize=11.7 K 2024-11-20T16:53:16,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/3b16e9aac3f0492da0738fcc4fe5f4ab as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/3b16e9aac3f0492da0738fcc4fe5f4ab 2024-11-20T16:53:16,566 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/3b16e9aac3f0492da0738fcc4fe5f4ab, entries=150, sequenceid=59, filesize=11.7 K 2024-11-20T16:53:16,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/36cff06bde1f433db6fee33f67b3dc21 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/36cff06bde1f433db6fee33f67b3dc21 2024-11-20T16:53:16,595 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/36cff06bde1f433db6fee33f67b3dc21, entries=150, sequenceid=59, filesize=11.7 K 2024-11-20T16:53:16,596 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e1410d95b5f65cde4303e83d1cb47b81 in 617ms, sequenceid=59, compaction requested=true 2024-11-20T16:53:16,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:16,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:16,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-20T16:53:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-20T16:53:16,600 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-20T16:53:16,600 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0090 sec 2024-11-20T16:53:16,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.0210 sec 2024-11-20T16:53:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:16,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-20T16:53:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:16,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:16,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:16,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:16,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/9a8c431930644aef9902c6ae0d74bc52 is 50, key is test_row_0/A:col10/1732121596057/Put/seqid=0 2024-11-20T16:53:16,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742114_1290 (size=14341) 2024-11-20T16:53:16,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/9a8c431930644aef9902c6ae0d74bc52 2024-11-20T16:53:16,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/46e0cb50c2d84123ae7ff3669d10f515 is 50, key is test_row_0/B:col10/1732121596057/Put/seqid=0 2024-11-20T16:53:16,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121656691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121656692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45996 deadline: 1732121656696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732121656696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121656698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T16:53:16,737 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:16,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742115_1291 (size=12001) 2024-11-20T16:53:16,741 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-20T16:53:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T16:53:16,744 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:16,746 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:16,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:16,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121656806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121656806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121656816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T16:53:16,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T16:53:16,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:16,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:16,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:16,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:16,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121657016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121657016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121657026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T16:53:17,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T16:53:17,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:17,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:17,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:17,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/46e0cb50c2d84123ae7ff3669d10f515 2024-11-20T16:53:17,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/897b6534afdf4022a57faf2f36b21382 is 50, key is test_row_0/C:col10/1732121596057/Put/seqid=0 2024-11-20T16:53:17,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742116_1292 (size=12001) 2024-11-20T16:53:17,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/897b6534afdf4022a57faf2f36b21382 2024-11-20T16:53:17,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/9a8c431930644aef9902c6ae0d74bc52 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/9a8c431930644aef9902c6ae0d74bc52 2024-11-20T16:53:17,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/9a8c431930644aef9902c6ae0d74bc52, entries=200, sequenceid=94, filesize=14.0 K 2024-11-20T16:53:17,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/46e0cb50c2d84123ae7ff3669d10f515 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/46e0cb50c2d84123ae7ff3669d10f515 2024-11-20T16:53:17,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/46e0cb50c2d84123ae7ff3669d10f515, entries=150, sequenceid=94, filesize=11.7 K 2024-11-20T16:53:17,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/897b6534afdf4022a57faf2f36b21382 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/897b6534afdf4022a57faf2f36b21382 2024-11-20T16:53:17,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/897b6534afdf4022a57faf2f36b21382, entries=150, sequenceid=94, filesize=11.7 K 2024-11-20T16:53:17,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for e1410d95b5f65cde4303e83d1cb47b81 in 543ms, sequenceid=94, compaction requested=true 2024-11-20T16:53:17,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:17,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:53:17,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:17,203 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:53:17,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:53:17,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:17,203 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:53:17,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:53:17,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:17,204 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:53:17,204 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/B is initiating minor compaction (all files) 2024-11-20T16:53:17,204 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/B in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:17,205 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/3d53b08e15824c7fa5495927a7015c45, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/306ffc6c4e9c4ad28736484a0ea665e2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/3b16e9aac3f0492da0738fcc4fe5f4ab, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/46e0cb50c2d84123ae7ff3669d10f515] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=46.9 K 2024-11-20T16:53:17,205 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:53:17,205 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/A is initiating minor compaction (all files) 2024-11-20T16:53:17,205 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/A in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:17,205 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/fa2a0e8e70aa40a1a024d6eaf7079c5a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/7c6ab23c4b7943508516703f4181559e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/ce8c5af24dbb4a1294a695091f2b7e4a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/9a8c431930644aef9902c6ae0d74bc52] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=51.4 K 2024-11-20T16:53:17,205 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d53b08e15824c7fa5495927a7015c45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732121594591 2024-11-20T16:53:17,215 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa2a0e8e70aa40a1a024d6eaf7079c5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732121594591 2024-11-20T16:53:17,216 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 306ffc6c4e9c4ad28736484a0ea665e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732121594680 2024-11-20T16:53:17,217 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b16e9aac3f0492da0738fcc4fe5f4ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732121595346 2024-11-20T16:53:17,217 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c6ab23c4b7943508516703f4181559e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732121594680 2024-11-20T16:53:17,217 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 46e0cb50c2d84123ae7ff3669d10f515, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732121596052 2024-11-20T16:53:17,217 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce8c5af24dbb4a1294a695091f2b7e4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732121595346 2024-11-20T16:53:17,218 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a8c431930644aef9902c6ae0d74bc52, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732121596050 2024-11-20T16:53:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:17,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T16:53:17,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:17,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:17,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:17,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:17,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:17,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:17,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/f773a7218dcd4fb8acbe8b6a045f84bd is 50, key is test_row_1/A:col10/1732121597218/Put/seqid=0 2024-11-20T16:53:17,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T16:53:17,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:17,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:17,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:17,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,246 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#B#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:17,247 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/07b1719fe87d4415b4cc1b29add2835b is 50, key is test_row_0/B:col10/1732121596057/Put/seqid=0 2024-11-20T16:53:17,250 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#A#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:17,250 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/49c3e13ac8a54fa4a2a51d21b55b0b5e is 50, key is test_row_0/A:col10/1732121596057/Put/seqid=0 2024-11-20T16:53:17,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742117_1293 (size=11997) 2024-11-20T16:53:17,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121597255, max timestamp=1732121597278Num uniques 859; 2024-11-20T16:53:17,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:53:17,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121597255, max timestamp=1732121597278Num uniques 859; 2024-11-20T16:53:17,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:53:17,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121597255, max timestamp=1732121597278Num uniques 859; 2024-11-20T16:53:17,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:53:17,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742119_1295 (size=12139) 2024-11-20T16:53:17,305 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/07b1719fe87d4415b4cc1b29add2835b as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/07b1719fe87d4415b4cc1b29add2835b 2024-11-20T16:53:17,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742118_1294 (size=12139) 2024-11-20T16:53:17,312 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/B of e1410d95b5f65cde4303e83d1cb47b81 into 07b1719fe87d4415b4cc1b29add2835b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:17,312 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:17,312 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/B, priority=12, startTime=1732121597203; duration=0sec 2024-11-20T16:53:17,312 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:17,312 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:B 2024-11-20T16:53:17,313 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:53:17,315 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:53:17,315 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/C is initiating minor compaction (all files) 2024-11-20T16:53:17,315 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/C in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:17,315 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/03df61dfd79244448c1bdd4642cb145e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/b2bd8c3d505a4d50a2a005b64409607f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/36cff06bde1f433db6fee33f67b3dc21, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/897b6534afdf4022a57faf2f36b21382] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=46.9 K 2024-11-20T16:53:17,316 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 03df61dfd79244448c1bdd4642cb145e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732121594591 2024-11-20T16:53:17,317 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting b2bd8c3d505a4d50a2a005b64409607f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732121594680 2024-11-20T16:53:17,317 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 36cff06bde1f433db6fee33f67b3dc21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732121595346 2024-11-20T16:53:17,318 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 897b6534afdf4022a57faf2f36b21382, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732121596052 2024-11-20T16:53:17,340 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#C#compaction#242 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:17,341 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/8d674cc39f874d189398a6802ddd60da is 50, key is test_row_0/C:col10/1732121596057/Put/seqid=0 2024-11-20T16:53:17,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121657337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45996 deadline: 1732121657337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121657340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121657340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732121657346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T16:53:17,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742120_1296 (size=12139) 2024-11-20T16:53:17,381 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/8d674cc39f874d189398a6802ddd60da as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/8d674cc39f874d189398a6802ddd60da 2024-11-20T16:53:17,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T16:53:17,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:17,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:17,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:17,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,412 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/C of e1410d95b5f65cde4303e83d1cb47b81 into 8d674cc39f874d189398a6802ddd60da(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:17,413 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:17,413 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/C, priority=12, startTime=1732121597203; duration=0sec 2024-11-20T16:53:17,413 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:17,413 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:C 2024-11-20T16:53:17,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121657456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45996 deadline: 1732121657457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121657457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732121657457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T16:53:17,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:17,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:17,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:17,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,579 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T16:53:17,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/f773a7218dcd4fb8acbe8b6a045f84bd 2024-11-20T16:53:17,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45996 deadline: 1732121657667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121657667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121657667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732121657676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/8b1a44d106664feba6388713481eebb5 is 50, key is test_row_1/B:col10/1732121597218/Put/seqid=0 2024-11-20T16:53:17,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T16:53:17,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:17,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:17,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,743 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/49c3e13ac8a54fa4a2a51d21b55b0b5e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/49c3e13ac8a54fa4a2a51d21b55b0b5e 2024-11-20T16:53:17,751 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/A of e1410d95b5f65cde4303e83d1cb47b81 into 49c3e13ac8a54fa4a2a51d21b55b0b5e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:17,751 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:17,751 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/A, priority=12, startTime=1732121597203; duration=0sec 2024-11-20T16:53:17,751 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:17,751 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:A 2024-11-20T16:53:17,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742121_1297 (size=9657) 2024-11-20T16:53:17,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121657856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T16:53:17,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T16:53:17,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:17,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:17,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:17,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:17,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45996 deadline: 1732121657976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121657976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121657977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:17,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732121657986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:17,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:18,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T16:53:18,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:18,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:18,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:18,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:18,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:18,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/8b1a44d106664feba6388713481eebb5 2024-11-20T16:53:18,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/70e9cd6f6635414d9ccb3ad30bf52c16 is 50, key is test_row_1/C:col10/1732121597218/Put/seqid=0 2024-11-20T16:53:18,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T16:53:18,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:18,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:18,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:18,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:18,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:18,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742122_1298 (size=9657) 2024-11-20T16:53:18,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T16:53:18,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:18,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:18,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:18,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:18,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:18,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121658486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:18,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45996 deadline: 1732121658486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:18,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121658486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:18,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T16:53:18,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:18,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:18,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:18,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:18,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:18,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732121658497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:18,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/70e9cd6f6635414d9ccb3ad30bf52c16 2024-11-20T16:53:18,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/f773a7218dcd4fb8acbe8b6a045f84bd as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/f773a7218dcd4fb8acbe8b6a045f84bd 2024-11-20T16:53:18,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T16:53:18,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:18,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:18,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:18,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:18,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:18,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/f773a7218dcd4fb8acbe8b6a045f84bd, entries=150, sequenceid=106, filesize=11.7 K 2024-11-20T16:53:18,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/8b1a44d106664feba6388713481eebb5 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/8b1a44d106664feba6388713481eebb5 2024-11-20T16:53:18,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/8b1a44d106664feba6388713481eebb5, entries=100, sequenceid=106, filesize=9.4 K 2024-11-20T16:53:18,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/70e9cd6f6635414d9ccb3ad30bf52c16 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/70e9cd6f6635414d9ccb3ad30bf52c16 2024-11-20T16:53:18,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/70e9cd6f6635414d9ccb3ad30bf52c16, entries=100, sequenceid=106, filesize=9.4 K 2024-11-20T16:53:18,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for e1410d95b5f65cde4303e83d1cb47b81 in 1457ms, sequenceid=106, compaction requested=false 2024-11-20T16:53:18,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:18,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T16:53:18,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:18,809 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-20T16:53:18,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:18,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:18,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:18,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:18,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:18,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:18,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/0e4b9bb30a3a4814a308d768f5d2c611 is 50, key is test_row_0/A:col10/1732121597334/Put/seqid=0 2024-11-20T16:53:18,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742123_1299 (size=12151) 2024-11-20T16:53:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:18,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T16:53:18,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121658930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:18,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:19,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121659046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:19,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:19,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121659257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:19,281 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/0e4b9bb30a3a4814a308d768f5d2c611 2024-11-20T16:53:19,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/6837ae935c1646b6b4e04a2c4cf2923a is 50, key is test_row_0/B:col10/1732121597334/Put/seqid=0 2024-11-20T16:53:19,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742124_1300 (size=12151) 2024-11-20T16:53:19,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121659496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:19,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45996 deadline: 1732121659496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:19,498 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_0' in e1410d95b5f65cde4303e83d1cb47b81 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:19,499 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in e1410d95b5f65cde4303e83d1cb47b81 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:19,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121659506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:19,508 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in e1410d95b5f65cde4303e83d1cb47b81 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:19,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:19,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:19,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:19,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732121659516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:19,521 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_0' in e1410d95b5f65cde4303e83d1cb47b81 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:19,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121659566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:19,722 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/6837ae935c1646b6b4e04a2c4cf2923a 2024-11-20T16:53:19,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/034cb435ceff4b6e84c326e739d49008 is 50, key is test_row_0/C:col10/1732121597334/Put/seqid=0 2024-11-20T16:53:19,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742125_1301 (size=12151) 2024-11-20T16:53:20,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121660076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:20,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:20,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:20,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:20,165 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/034cb435ceff4b6e84c326e739d49008 2024-11-20T16:53:20,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/0e4b9bb30a3a4814a308d768f5d2c611 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/0e4b9bb30a3a4814a308d768f5d2c611 2024-11-20T16:53:20,177 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/0e4b9bb30a3a4814a308d768f5d2c611, entries=150, sequenceid=144, filesize=11.9 K 2024-11-20T16:53:20,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/6837ae935c1646b6b4e04a2c4cf2923a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/6837ae935c1646b6b4e04a2c4cf2923a 2024-11-20T16:53:20,183 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/6837ae935c1646b6b4e04a2c4cf2923a, entries=150, sequenceid=144, filesize=11.9 K 2024-11-20T16:53:20,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/034cb435ceff4b6e84c326e739d49008 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/034cb435ceff4b6e84c326e739d49008 2024-11-20T16:53:20,190 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/034cb435ceff4b6e84c326e739d49008, entries=150, sequenceid=144, filesize=11.9 K 2024-11-20T16:53:20,193 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for e1410d95b5f65cde4303e83d1cb47b81 in 1384ms, sequenceid=144, compaction requested=true 2024-11-20T16:53:20,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:20,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:20,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-20T16:53:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-20T16:53:20,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-20T16:53:20,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4490 sec 2024-11-20T16:53:20,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 3.4570 sec 2024-11-20T16:53:20,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T16:53:20,896 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:20,896 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-20T16:53:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T16:53:20,898 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:20,899 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:20,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T16:53:21,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T16:53:21,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:21,054 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T16:53:21,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:21,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:21,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:21,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:21,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:21,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:21,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/5cc7481fe206407bba40aea6383d59d5 is 50, key is test_row_0/A:col10/1732121598923/Put/seqid=0 2024-11-20T16:53:21,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742126_1302 (size=12151) 2024-11-20T16:53:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:21,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:21,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121601167, max timestamp=1732121601192Num uniques 859; 2024-11-20T16:53:21,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:53:21,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121601167, max timestamp=1732121601192Num uniques 859; 2024-11-20T16:53:21,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121601167, max timestamp=1732121601192Num uniques 859; 2024-11-20T16:53:21,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:53:21,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:53:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T16:53:21,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:21,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121661295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:21,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:21,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:21,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:21,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121661406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:21,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:21,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:21,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:21,482 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/5cc7481fe206407bba40aea6383d59d5 2024-11-20T16:53:21,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/2b5a893f1eca43f9a376360fb607ace2 is 50, key is test_row_0/B:col10/1732121598923/Put/seqid=0 2024-11-20T16:53:21,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742127_1303 (size=12151) 2024-11-20T16:53:21,505 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/2b5a893f1eca43f9a376360fb607ace2 2024-11-20T16:53:21,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/d863695033104cca8970150e3b08c44e is 50, key is test_row_0/C:col10/1732121598923/Put/seqid=0 2024-11-20T16:53:21,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:21,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121661516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:21,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:21,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45996 deadline: 1732121661516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:21,519 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_0' in e1410d95b5f65cde4303e83d1cb47b81 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:21,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:21,520 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in e1410d95b5f65cde4303e83d1cb47b81 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:21,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:21,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:21,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:21,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:21,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:21,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742128_1304 (size=12151) 2024-11-20T16:53:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T16:53:21,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121661537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:21,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732121661537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:21,541 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in e1410d95b5f65cde4303e83d1cb47b81 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:21,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:21,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:21,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:21,542 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_0' in e1410d95b5f65cde4303e83d1cb47b81 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:21,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121661617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:21,924 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/d863695033104cca8970150e3b08c44e 2024-11-20T16:53:21,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121661926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:21,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/5cc7481fe206407bba40aea6383d59d5 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/5cc7481fe206407bba40aea6383d59d5 2024-11-20T16:53:21,934 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/5cc7481fe206407bba40aea6383d59d5, entries=150, sequenceid=155, filesize=11.9 K 2024-11-20T16:53:21,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/2b5a893f1eca43f9a376360fb607ace2 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/2b5a893f1eca43f9a376360fb607ace2 2024-11-20T16:53:21,940 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/2b5a893f1eca43f9a376360fb607ace2, entries=150, sequenceid=155, filesize=11.9 K 2024-11-20T16:53:21,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/d863695033104cca8970150e3b08c44e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/d863695033104cca8970150e3b08c44e 2024-11-20T16:53:21,947 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/d863695033104cca8970150e3b08c44e, entries=150, sequenceid=155, filesize=11.9 K 2024-11-20T16:53:21,948 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e1410d95b5f65cde4303e83d1cb47b81 in 894ms, sequenceid=155, compaction requested=true 2024-11-20T16:53:21,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:21,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:21,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-20T16:53:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-20T16:53:21,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-20T16:53:21,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0520 sec 2024-11-20T16:53:21,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.0580 sec 2024-11-20T16:53:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T16:53:22,036 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:22,037 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-20T16:53:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T16:53:22,039 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:22,040 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:22,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T16:53:22,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T16:53:22,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:22,192 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-20T16:53:22,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:22,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:22,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:22,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:22,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:22,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:22,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/badfa826877c4706ac7cfd6d12dc17ee is 50, key is test_row_0/A:col10/1732121601293/Put/seqid=0 2024-11-20T16:53:22,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742129_1305 (size=12151) 2024-11-20T16:53:22,204 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/badfa826877c4706ac7cfd6d12dc17ee 2024-11-20T16:53:22,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/f0cc980a68974b7fa7a83bb49dcd0b1e is 50, key is test_row_0/B:col10/1732121601293/Put/seqid=0 2024-11-20T16:53:22,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742130_1306 (size=12151) 2024-11-20T16:53:22,247 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/f0cc980a68974b7fa7a83bb49dcd0b1e 2024-11-20T16:53:22,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/b94412bde93e407eade0670582faf4fc is 50, key is test_row_0/C:col10/1732121601293/Put/seqid=0 2024-11-20T16:53:22,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742131_1307 (size=12151) 2024-11-20T16:53:22,295 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/b94412bde93e407eade0670582faf4fc 2024-11-20T16:53:22,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/badfa826877c4706ac7cfd6d12dc17ee as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/badfa826877c4706ac7cfd6d12dc17ee 2024-11-20T16:53:22,309 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/badfa826877c4706ac7cfd6d12dc17ee, entries=150, sequenceid=191, filesize=11.9 K 2024-11-20T16:53:22,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/f0cc980a68974b7fa7a83bb49dcd0b1e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/f0cc980a68974b7fa7a83bb49dcd0b1e 2024-11-20T16:53:22,315 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/f0cc980a68974b7fa7a83bb49dcd0b1e, entries=150, sequenceid=191, filesize=11.9 K 2024-11-20T16:53:22,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/b94412bde93e407eade0670582faf4fc as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/b94412bde93e407eade0670582faf4fc 2024-11-20T16:53:22,335 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/b94412bde93e407eade0670582faf4fc, entries=150, sequenceid=191, filesize=11.9 K 2024-11-20T16:53:22,339 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=0 B/0 for e1410d95b5f65cde4303e83d1cb47b81 in 147ms, sequenceid=191, compaction requested=true 2024-11-20T16:53:22,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:22,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:22,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-20T16:53:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-20T16:53:22,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-20T16:53:22,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 302 msec 2024-11-20T16:53:22,350 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 310 msec 2024-11-20T16:53:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T16:53:22,357 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:22,358 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-20T16:53:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T16:53:22,362 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:22,364 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:22,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:22,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T16:53:22,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:22,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:22,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:22,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:22,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:22,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:22,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/2b767b5c161d48338bdf6edbc3e16162 is 50, key is test_row_0/A:col10/1732121602452/Put/seqid=0 2024-11-20T16:53:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T16:53:22,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742132_1308 (size=16931) 2024-11-20T16:53:22,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T16:53:22,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:22,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:22,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:22,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:22,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:22,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121602497, max timestamp=1732121602568Num uniques 859; 2024-11-20T16:53:22,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121602497, max timestamp=1732121602568Num uniques 859; 2024-11-20T16:53:22,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:53:22,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:53:22,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121602497, max timestamp=1732121602568Num uniques 859; 2024-11-20T16:53:22,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:53:22,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T16:53:22,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:22,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:22,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:22,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:22,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T16:53:22,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121662697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:22,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:22,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:22,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:22,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121662815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:22,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T16:53:22,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:22,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:22,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:22,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:22,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:22,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/2b767b5c161d48338bdf6edbc3e16162 2024-11-20T16:53:22,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/19e489e0778243349917979508813ae4 is 50, key is test_row_0/B:col10/1732121602452/Put/seqid=0 2024-11-20T16:53:22,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742133_1309 (size=12151) 2024-11-20T16:53:22,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/19e489e0778243349917979508813ae4 2024-11-20T16:53:22,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/76fc5c2872d5412d86089bfa6d1feecb is 50, key is test_row_0/C:col10/1732121602452/Put/seqid=0 2024-11-20T16:53:22,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742134_1310 (size=12151) 2024-11-20T16:53:22,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/76fc5c2872d5412d86089bfa6d1feecb 2024-11-20T16:53:22,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/2b767b5c161d48338bdf6edbc3e16162 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/2b767b5c161d48338bdf6edbc3e16162 2024-11-20T16:53:22,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/2b767b5c161d48338bdf6edbc3e16162, entries=250, sequenceid=201, filesize=16.5 K 2024-11-20T16:53:22,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/19e489e0778243349917979508813ae4 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/19e489e0778243349917979508813ae4 2024-11-20T16:53:22,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/19e489e0778243349917979508813ae4, entries=150, sequenceid=201, filesize=11.9 K 2024-11-20T16:53:22,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/76fc5c2872d5412d86089bfa6d1feecb as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/76fc5c2872d5412d86089bfa6d1feecb 2024-11-20T16:53:22,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/76fc5c2872d5412d86089bfa6d1feecb, entries=150, sequenceid=201, filesize=11.9 K 2024-11-20T16:53:22,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e1410d95b5f65cde4303e83d1cb47b81 in 517ms, sequenceid=201, compaction requested=true 2024-11-20T16:53:22,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:22,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:53:22,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:22,971 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-20T16:53:22,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:53:22,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:22,971 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-20T16:53:22,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:53:22,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:22,972 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 77520 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-20T16:53:22,972 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/A is initiating minor compaction (all files) 2024-11-20T16:53:22,972 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/A in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:22,972 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/49c3e13ac8a54fa4a2a51d21b55b0b5e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/f773a7218dcd4fb8acbe8b6a045f84bd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/0e4b9bb30a3a4814a308d768f5d2c611, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/5cc7481fe206407bba40aea6383d59d5, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/badfa826877c4706ac7cfd6d12dc17ee, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/2b767b5c161d48338bdf6edbc3e16162] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=75.7 K 2024-11-20T16:53:22,973 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 70400 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-20T16:53:22,973 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/B is initiating minor compaction (all files) 2024-11-20T16:53:22,973 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/B in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:22,973 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/07b1719fe87d4415b4cc1b29add2835b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/8b1a44d106664feba6388713481eebb5, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/6837ae935c1646b6b4e04a2c4cf2923a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/2b5a893f1eca43f9a376360fb607ace2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/f0cc980a68974b7fa7a83bb49dcd0b1e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/19e489e0778243349917979508813ae4] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=68.8 K 2024-11-20T16:53:22,974 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49c3e13ac8a54fa4a2a51d21b55b0b5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732121596052 2024-11-20T16:53:22,974 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 07b1719fe87d4415b4cc1b29add2835b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732121596052 2024-11-20T16:53:22,974 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting f773a7218dcd4fb8acbe8b6a045f84bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732121596692 2024-11-20T16:53:22,974 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e4b9bb30a3a4814a308d768f5d2c611, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732121597334 2024-11-20T16:53:22,975 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cc7481fe206407bba40aea6383d59d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732121598907 2024-11-20T16:53:22,975 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting badfa826877c4706ac7cfd6d12dc17ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732121601280 2024-11-20T16:53:22,975 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b1a44d106664feba6388713481eebb5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732121597218 2024-11-20T16:53:22,975 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b767b5c161d48338bdf6edbc3e16162, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732121602437 2024-11-20T16:53:22,976 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 6837ae935c1646b6b4e04a2c4cf2923a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732121597334 2024-11-20T16:53:22,976 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b5a893f1eca43f9a376360fb607ace2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732121598907 2024-11-20T16:53:22,977 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting f0cc980a68974b7fa7a83bb49dcd0b1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732121601280 2024-11-20T16:53:22,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T16:53:22,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:22,977 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-20T16:53:22,978 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 19e489e0778243349917979508813ae4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732121602437 2024-11-20T16:53:22,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:22,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:22,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:22,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:22,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:22,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:22,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/d35d13bdf5ca4287afd6eef95c160c3b is 50, key is test_row_0/A:col10/1732121602641/Put/seqid=0 2024-11-20T16:53:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T16:53:23,003 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#A#compaction#258 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:23,003 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/409c36ec1e394e44b9ca6b499305a08e is 50, key is test_row_0/A:col10/1732121602452/Put/seqid=0 2024-11-20T16:53:23,012 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#B#compaction#259 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:23,013 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/636e8a7aad324633bae88d30614a3692 is 50, key is test_row_0/B:col10/1732121602452/Put/seqid=0 2024-11-20T16:53:23,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742135_1311 (size=12151) 2024-11-20T16:53:23,018 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/d35d13bdf5ca4287afd6eef95c160c3b 2024-11-20T16:53:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:23,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/b81b0c132b41434a993d150bb11cabd1 is 50, key is test_row_0/B:col10/1732121602641/Put/seqid=0 2024-11-20T16:53:23,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742136_1312 (size=12493) 2024-11-20T16:53:23,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742137_1313 (size=12493) 2024-11-20T16:53:23,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742138_1314 (size=12151) 2024-11-20T16:53:23,068 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/b81b0c132b41434a993d150bb11cabd1 2024-11-20T16:53:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/e0f5e760736d4753a9f14df7e9adc910 is 50, key is test_row_0/C:col10/1732121602641/Put/seqid=0 2024-11-20T16:53:23,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742139_1315 (size=12151) 2024-11-20T16:53:23,084 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/e0f5e760736d4753a9f14df7e9adc910 2024-11-20T16:53:23,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121663083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:23,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:23,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:23,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:23,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/d35d13bdf5ca4287afd6eef95c160c3b as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/d35d13bdf5ca4287afd6eef95c160c3b 2024-11-20T16:53:23,095 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/d35d13bdf5ca4287afd6eef95c160c3b, entries=150, sequenceid=238, filesize=11.9 K 2024-11-20T16:53:23,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/b81b0c132b41434a993d150bb11cabd1 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/b81b0c132b41434a993d150bb11cabd1 2024-11-20T16:53:23,100 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/b81b0c132b41434a993d150bb11cabd1, entries=150, sequenceid=238, filesize=11.9 K 2024-11-20T16:53:23,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/e0f5e760736d4753a9f14df7e9adc910 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/e0f5e760736d4753a9f14df7e9adc910 2024-11-20T16:53:23,107 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/e0f5e760736d4753a9f14df7e9adc910, entries=150, sequenceid=238, filesize=11.9 K 2024-11-20T16:53:23,108 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for e1410d95b5f65cde4303e83d1cb47b81 in 131ms, sequenceid=238, compaction requested=true 2024-11-20T16:53:23,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:23,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:23,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-20T16:53:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-20T16:53:23,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-20T16:53:23,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 746 msec 2024-11-20T16:53:23,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 756 msec 2024-11-20T16:53:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:23,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T16:53:23,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:23,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:23,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:23,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:23,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:23,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:23,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/42719accf1b241548ea76dbd8369693e is 50, key is test_row_0/A:col10/1732121603055/Put/seqid=0 2024-11-20T16:53:23,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742140_1316 (size=14541) 2024-11-20T16:53:23,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/42719accf1b241548ea76dbd8369693e 2024-11-20T16:53:23,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/1c24b6745f834e839046ee7823ff48b2 is 50, key is test_row_0/B:col10/1732121603055/Put/seqid=0 2024-11-20T16:53:23,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742141_1317 (size=12151) 2024-11-20T16:53:23,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/1c24b6745f834e839046ee7823ff48b2 2024-11-20T16:53:23,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/eb883026bfdc47c1807f58db034233c1 is 50, key is test_row_0/C:col10/1732121603055/Put/seqid=0 2024-11-20T16:53:23,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742142_1318 (size=12151) 2024-11-20T16:53:23,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121603322, max timestamp=1732121603350Num uniques 859; 2024-11-20T16:53:23,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:53:23,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121603322, max timestamp=1732121603350Num uniques 859; 2024-11-20T16:53:23,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121603322, max timestamp=1732121603350Num uniques 859; 2024-11-20T16:53:23,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:53:23,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:53:23,464 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/409c36ec1e394e44b9ca6b499305a08e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/409c36ec1e394e44b9ca6b499305a08e 2024-11-20T16:53:23,467 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/636e8a7aad324633bae88d30614a3692 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/636e8a7aad324633bae88d30614a3692 2024-11-20T16:53:23,469 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/A of e1410d95b5f65cde4303e83d1cb47b81 into 409c36ec1e394e44b9ca6b499305a08e(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:23,469 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:23,469 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/A, priority=10, startTime=1732121602970; duration=0sec 2024-11-20T16:53:23,470 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:23,470 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:A 2024-11-20T16:53:23,470 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-20T16:53:23,473 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/B of e1410d95b5f65cde4303e83d1cb47b81 into 636e8a7aad324633bae88d30614a3692(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:23,473 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:23,473 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/B, priority=10, startTime=1732121602971; duration=0sec 2024-11-20T16:53:23,473 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:23,473 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:B 2024-11-20T16:53:23,474 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 82551 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-20T16:53:23,474 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/C is initiating minor compaction (all files) 2024-11-20T16:53:23,474 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/C in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:23,474 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/8d674cc39f874d189398a6802ddd60da, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/70e9cd6f6635414d9ccb3ad30bf52c16, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/034cb435ceff4b6e84c326e739d49008, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/d863695033104cca8970150e3b08c44e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/b94412bde93e407eade0670582faf4fc, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/76fc5c2872d5412d86089bfa6d1feecb, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/e0f5e760736d4753a9f14df7e9adc910] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=80.6 K 2024-11-20T16:53:23,475 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d674cc39f874d189398a6802ddd60da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732121596052 2024-11-20T16:53:23,475 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70e9cd6f6635414d9ccb3ad30bf52c16, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732121597218 2024-11-20T16:53:23,475 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 034cb435ceff4b6e84c326e739d49008, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732121597334 2024-11-20T16:53:23,476 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting d863695033104cca8970150e3b08c44e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732121598907 2024-11-20T16:53:23,476 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting b94412bde93e407eade0670582faf4fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732121601280 2024-11-20T16:53:23,476 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76fc5c2872d5412d86089bfa6d1feecb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732121602437 2024-11-20T16:53:23,477 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0f5e760736d4753a9f14df7e9adc910, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732121602641 2024-11-20T16:53:23,490 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#C#compaction#265 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:23,490 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/0333f28624a3465886083b4d34516b99 is 50, key is test_row_0/C:col10/1732121602641/Put/seqid=0 2024-11-20T16:53:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T16:53:23,496 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:23,497 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-20T16:53:23,499 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:23,503 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:23,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T16:53:23,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121663505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:23,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742143_1319 (size=12527) 2024-11-20T16:53:23,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:23,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:23,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:23,518 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/0333f28624a3465886083b4d34516b99 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/0333f28624a3465886083b4d34516b99 2024-11-20T16:53:23,524 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/C of e1410d95b5f65cde4303e83d1cb47b81 into 0333f28624a3465886083b4d34516b99(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:23,524 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:23,524 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/C, priority=9, startTime=1732121602971; duration=0sec 2024-11-20T16:53:23,525 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:23,525 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:C 2024-11-20T16:53:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T16:53:23,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121663615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:23,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/eb883026bfdc47c1807f58db034233c1 2024-11-20T16:53:23,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/42719accf1b241548ea76dbd8369693e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/42719accf1b241548ea76dbd8369693e 2024-11-20T16:53:23,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T16:53:23,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:23,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:23,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:23,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:23,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:23,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/42719accf1b241548ea76dbd8369693e, entries=200, sequenceid=249, filesize=14.2 K 2024-11-20T16:53:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:23,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/1c24b6745f834e839046ee7823ff48b2 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/1c24b6745f834e839046ee7823ff48b2 2024-11-20T16:53:23,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/1c24b6745f834e839046ee7823ff48b2, entries=150, sequenceid=249, filesize=11.9 K 2024-11-20T16:53:23,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/eb883026bfdc47c1807f58db034233c1 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/eb883026bfdc47c1807f58db034233c1 2024-11-20T16:53:23,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/eb883026bfdc47c1807f58db034233c1, entries=150, sequenceid=249, filesize=11.9 K 2024-11-20T16:53:23,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e1410d95b5f65cde4303e83d1cb47b81 in 474ms, sequenceid=249, compaction requested=true 2024-11-20T16:53:23,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:23,672 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:23,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:53:23,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:23,672 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:23,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:53:23,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:23,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:53:23,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:23,673 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:23,673 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/A is initiating minor compaction (all files) 2024-11-20T16:53:23,673 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:23,674 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/A in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:23,674 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/B is initiating minor compaction (all files) 2024-11-20T16:53:23,674 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/B in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:23,674 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/409c36ec1e394e44b9ca6b499305a08e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/d35d13bdf5ca4287afd6eef95c160c3b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/42719accf1b241548ea76dbd8369693e] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=38.3 K 2024-11-20T16:53:23,674 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/636e8a7aad324633bae88d30614a3692, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/b81b0c132b41434a993d150bb11cabd1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/1c24b6745f834e839046ee7823ff48b2] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=35.9 K 2024-11-20T16:53:23,675 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 636e8a7aad324633bae88d30614a3692, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732121602437 2024-11-20T16:53:23,675 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 409c36ec1e394e44b9ca6b499305a08e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732121602437 2024-11-20T16:53:23,675 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting d35d13bdf5ca4287afd6eef95c160c3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732121602641 2024-11-20T16:53:23,675 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting b81b0c132b41434a993d150bb11cabd1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732121602641 2024-11-20T16:53:23,675 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 42719accf1b241548ea76dbd8369693e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732121603055 2024-11-20T16:53:23,675 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c24b6745f834e839046ee7823ff48b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732121603055 2024-11-20T16:53:23,686 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#B#compaction#266 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:23,687 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/95ec9283d4eb4b669c67985b9a024bfb is 50, key is test_row_0/B:col10/1732121603055/Put/seqid=0 2024-11-20T16:53:23,693 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#A#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:23,693 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/3b004acbf46346e1b115701e88b0702a is 50, key is test_row_0/A:col10/1732121603055/Put/seqid=0 2024-11-20T16:53:23,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742144_1320 (size=12595) 2024-11-20T16:53:23,720 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/95ec9283d4eb4b669c67985b9a024bfb as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/95ec9283d4eb4b669c67985b9a024bfb 2024-11-20T16:53:23,728 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/B of e1410d95b5f65cde4303e83d1cb47b81 into 95ec9283d4eb4b669c67985b9a024bfb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:23,728 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:23,728 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/B, priority=13, startTime=1732121603672; duration=0sec 2024-11-20T16:53:23,729 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:23,729 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:B 2024-11-20T16:53:23,729 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T16:53:23,731 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T16:53:23,732 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T16:53:23,732 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. because compaction request was cancelled 2024-11-20T16:53:23,732 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:C 2024-11-20T16:53:23,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742145_1321 (size=12595) 2024-11-20T16:53:23,746 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/3b004acbf46346e1b115701e88b0702a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/3b004acbf46346e1b115701e88b0702a 2024-11-20T16:53:23,752 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/A of e1410d95b5f65cde4303e83d1cb47b81 into 3b004acbf46346e1b115701e88b0702a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:23,752 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:23,752 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/A, priority=13, startTime=1732121603672; duration=0sec 2024-11-20T16:53:23,752 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:23,752 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:A 2024-11-20T16:53:23,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T16:53:23,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:23,811 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-20T16:53:23,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:23,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:23,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:23,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:23,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:23,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T16:53:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:23,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/1ecb8b94afe14833a3f20004d6e232a8 is 50, key is test_row_0/A:col10/1732121603493/Put/seqid=0 2024-11-20T16:53:23,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:23,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742146_1322 (size=12301) 2024-11-20T16:53:23,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121663887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:23,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121663996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T16:53:24,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:24,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121664206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:24,253 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/1ecb8b94afe14833a3f20004d6e232a8 2024-11-20T16:53:24,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/d3057477cb5d4e0aa109af5cf62a7e58 is 50, key is test_row_0/B:col10/1732121603493/Put/seqid=0 2024-11-20T16:53:24,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742147_1323 (size=12301) 2024-11-20T16:53:24,327 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/d3057477cb5d4e0aa109af5cf62a7e58 2024-11-20T16:53:24,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/2761c1edb9a5481f89bd17d3741d3213 is 50, key is test_row_0/C:col10/1732121603493/Put/seqid=0 2024-11-20T16:53:24,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742148_1324 (size=12301) 2024-11-20T16:53:24,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121664527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T16:53:24,811 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/2761c1edb9a5481f89bd17d3741d3213 2024-11-20T16:53:24,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/1ecb8b94afe14833a3f20004d6e232a8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/1ecb8b94afe14833a3f20004d6e232a8 2024-11-20T16:53:24,833 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/1ecb8b94afe14833a3f20004d6e232a8, entries=150, sequenceid=290, filesize=12.0 K 2024-11-20T16:53:24,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/d3057477cb5d4e0aa109af5cf62a7e58 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/d3057477cb5d4e0aa109af5cf62a7e58 2024-11-20T16:53:24,839 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/d3057477cb5d4e0aa109af5cf62a7e58, entries=150, sequenceid=290, filesize=12.0 K 2024-11-20T16:53:24,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/2761c1edb9a5481f89bd17d3741d3213 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/2761c1edb9a5481f89bd17d3741d3213 2024-11-20T16:53:24,845 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/2761c1edb9a5481f89bd17d3741d3213, entries=150, sequenceid=290, filesize=12.0 K 2024-11-20T16:53:24,846 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for e1410d95b5f65cde4303e83d1cb47b81 in 1035ms, sequenceid=290, compaction requested=true 2024-11-20T16:53:24,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:24,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:24,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-20T16:53:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-20T16:53:24,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-20T16:53:24,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3450 sec 2024-11-20T16:53:24,852 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.3540 sec 2024-11-20T16:53:25,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:25,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T16:53:25,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:25,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:25,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:25,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:25,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:25,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:25,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/2b077852616a4cf59af01a61050f01ec is 50, key is test_row_0/A:col10/1732121603885/Put/seqid=0 2024-11-20T16:53:25,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742149_1325 (size=14741) 2024-11-20T16:53:25,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121665192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:25,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:25,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121665306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:25,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/2b077852616a4cf59af01a61050f01ec 2024-11-20T16:53:25,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:25,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121665516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:25,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:25,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:25,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:25,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/a0aafed670424e8e85534ad9edaefb59 is 50, key is test_row_0/B:col10/1732121603885/Put/seqid=0 2024-11-20T16:53:25,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:25,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1732121665527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:25,527 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_0' in e1410d95b5f65cde4303e83d1cb47b81 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:25,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:25,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:25,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:25,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:25,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45996 deadline: 1732121665533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:25,534 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in e1410d95b5f65cde4303e83d1cb47b81 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:25,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:25,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:25,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:25,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742150_1326 (size=12301) 2024-11-20T16:53:25,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/a0aafed670424e8e85534ad9edaefb59 2024-11-20T16:53:25,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45988 deadline: 1732121665566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:25,567 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in e1410d95b5f65cde4303e83d1cb47b81 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:25,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732121665576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:25,579 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_0' in e1410d95b5f65cde4303e83d1cb47b81 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:25,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/7fa905805ef54af28f2926122c19f987 is 50, key is test_row_0/C:col10/1732121603885/Put/seqid=0 2024-11-20T16:53:25,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742151_1327 (size=12301) 2024-11-20T16:53:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T16:53:25,656 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:25,657 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-20T16:53:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T16:53:25,660 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:25,661 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:25,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T16:53:25,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T16:53:25,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:25,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:25,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:25,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:25,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:25,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121665826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:25,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:25,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:25,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:25,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T16:53:25,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:25,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:25,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:25,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:25,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:25,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:25,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T16:53:25,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/7fa905805ef54af28f2926122c19f987 2024-11-20T16:53:26,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/2b077852616a4cf59af01a61050f01ec as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/2b077852616a4cf59af01a61050f01ec 2024-11-20T16:53:26,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/2b077852616a4cf59af01a61050f01ec, entries=200, sequenceid=302, filesize=14.4 K 2024-11-20T16:53:26,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/a0aafed670424e8e85534ad9edaefb59 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/a0aafed670424e8e85534ad9edaefb59 2024-11-20T16:53:26,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/a0aafed670424e8e85534ad9edaefb59, entries=150, sequenceid=302, filesize=12.0 K 2024-11-20T16:53:26,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/7fa905805ef54af28f2926122c19f987 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/7fa905805ef54af28f2926122c19f987 2024-11-20T16:53:26,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/7fa905805ef54af28f2926122c19f987, entries=150, sequenceid=302, filesize=12.0 K 2024-11-20T16:53:26,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e1410d95b5f65cde4303e83d1cb47b81 in 966ms, sequenceid=302, compaction requested=true 2024-11-20T16:53:26,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:26,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:53:26,018 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:26,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:26,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:53:26,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:26,018 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:26,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:53:26,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:26,019 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39637 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:26,019 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/A is initiating minor compaction (all files) 2024-11-20T16:53:26,019 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/A in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:26,019 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/3b004acbf46346e1b115701e88b0702a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/1ecb8b94afe14833a3f20004d6e232a8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/2b077852616a4cf59af01a61050f01ec] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=38.7 K 2024-11-20T16:53:26,019 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b004acbf46346e1b115701e88b0702a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732121603055 2024-11-20T16:53:26,019 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:26,019 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/B is initiating minor compaction (all files) 2024-11-20T16:53:26,019 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/B in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:26,019 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/95ec9283d4eb4b669c67985b9a024bfb, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/d3057477cb5d4e0aa109af5cf62a7e58, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/a0aafed670424e8e85534ad9edaefb59] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=36.3 K 2024-11-20T16:53:26,020 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 95ec9283d4eb4b669c67985b9a024bfb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732121603055 2024-11-20T16:53:26,020 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ecb8b94afe14833a3f20004d6e232a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732121603473 2024-11-20T16:53:26,020 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting d3057477cb5d4e0aa109af5cf62a7e58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732121603473 2024-11-20T16:53:26,020 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b077852616a4cf59af01a61050f01ec, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732121603862 2024-11-20T16:53:26,021 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting a0aafed670424e8e85534ad9edaefb59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732121603879 2024-11-20T16:53:26,029 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#B#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:26,029 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#A#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:26,029 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/2bb9ad17b55c4c1dbaca9aa78316b118 is 50, key is test_row_0/A:col10/1732121603885/Put/seqid=0 2024-11-20T16:53:26,029 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/f87c648e4c3442069f0d8c40a63e66ee is 50, key is test_row_0/B:col10/1732121603885/Put/seqid=0 2024-11-20T16:53:26,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742152_1328 (size=12847) 2024-11-20T16:53:26,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742153_1329 (size=12847) 2024-11-20T16:53:26,068 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/f87c648e4c3442069f0d8c40a63e66ee as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/f87c648e4c3442069f0d8c40a63e66ee 2024-11-20T16:53:26,073 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/B of e1410d95b5f65cde4303e83d1cb47b81 into f87c648e4c3442069f0d8c40a63e66ee(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:26,073 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:26,073 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/B, priority=13, startTime=1732121606018; duration=0sec 2024-11-20T16:53:26,073 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:26,073 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:B 2024-11-20T16:53:26,073 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:53:26,074 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49280 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:53:26,074 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/C is initiating minor compaction (all files) 2024-11-20T16:53:26,074 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/C in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:26,074 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/0333f28624a3465886083b4d34516b99, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/eb883026bfdc47c1807f58db034233c1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/2761c1edb9a5481f89bd17d3741d3213, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/7fa905805ef54af28f2926122c19f987] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=48.1 K 2024-11-20T16:53:26,075 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 0333f28624a3465886083b4d34516b99, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732121602641 2024-11-20T16:53:26,075 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting eb883026bfdc47c1807f58db034233c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732121603055 2024-11-20T16:53:26,076 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 2761c1edb9a5481f89bd17d3741d3213, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732121603473 2024-11-20T16:53:26,077 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fa905805ef54af28f2926122c19f987, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732121603879 2024-11-20T16:53:26,086 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#C#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:26,087 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/ecca7eb36bfb4da98878def401b9e414 is 50, key is test_row_0/C:col10/1732121603885/Put/seqid=0 2024-11-20T16:53:26,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742154_1330 (size=12813) 2024-11-20T16:53:26,101 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/ecca7eb36bfb4da98878def401b9e414 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/ecca7eb36bfb4da98878def401b9e414 2024-11-20T16:53:26,105 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/C of e1410d95b5f65cde4303e83d1cb47b81 into ecca7eb36bfb4da98878def401b9e414(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:26,106 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:26,106 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/C, priority=12, startTime=1732121606018; duration=0sec 2024-11-20T16:53:26,106 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:26,106 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:C 2024-11-20T16:53:26,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T16:53:26,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:26,122 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T16:53:26,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:26,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:26,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:26,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:26,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:26,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:26,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/c82a7e7d93ac49fc8d74c94ee3b5548a is 50, key is test_row_0/A:col10/1732121605188/Put/seqid=0 2024-11-20T16:53:26,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742155_1331 (size=12301) 2024-11-20T16:53:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T16:53:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:26,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:26,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121666372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:26,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:26,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:26,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:26,462 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/2bb9ad17b55c4c1dbaca9aa78316b118 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/2bb9ad17b55c4c1dbaca9aa78316b118 2024-11-20T16:53:26,467 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/A of e1410d95b5f65cde4303e83d1cb47b81 into 2bb9ad17b55c4c1dbaca9aa78316b118(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:26,467 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:26,467 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/A, priority=13, startTime=1732121606018; duration=0sec 2024-11-20T16:53:26,467 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:26,467 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:A 2024-11-20T16:53:26,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121666486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:26,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:26,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:26,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:26,553 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/c82a7e7d93ac49fc8d74c94ee3b5548a 2024-11-20T16:53:26,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/710e79f9e4134ff894095587e5c3548a is 50, key is test_row_0/B:col10/1732121605188/Put/seqid=0 2024-11-20T16:53:26,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742156_1332 (size=12301) 2024-11-20T16:53:26,575 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/710e79f9e4134ff894095587e5c3548a 2024-11-20T16:53:26,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/60b762ec1c164114a70c8fd4c70f9fc9 is 50, key is test_row_0/C:col10/1732121605188/Put/seqid=0 2024-11-20T16:53:26,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742157_1333 (size=12301) 2024-11-20T16:53:26,598 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/60b762ec1c164114a70c8fd4c70f9fc9 2024-11-20T16:53:26,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/c82a7e7d93ac49fc8d74c94ee3b5548a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/c82a7e7d93ac49fc8d74c94ee3b5548a 2024-11-20T16:53:26,606 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/c82a7e7d93ac49fc8d74c94ee3b5548a, entries=150, sequenceid=324, filesize=12.0 K 2024-11-20T16:53:26,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/710e79f9e4134ff894095587e5c3548a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/710e79f9e4134ff894095587e5c3548a 2024-11-20T16:53:26,614 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/710e79f9e4134ff894095587e5c3548a, entries=150, sequenceid=324, filesize=12.0 K 2024-11-20T16:53:26,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/60b762ec1c164114a70c8fd4c70f9fc9 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/60b762ec1c164114a70c8fd4c70f9fc9 2024-11-20T16:53:26,621 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/60b762ec1c164114a70c8fd4c70f9fc9, entries=150, sequenceid=324, filesize=12.0 K 2024-11-20T16:53:26,622 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e1410d95b5f65cde4303e83d1cb47b81 in 500ms, sequenceid=324, compaction requested=false 2024-11-20T16:53:26,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:26,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:26,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-20T16:53:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-20T16:53:26,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-20T16:53:26,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 962 msec 2024-11-20T16:53:26,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 970 msec 2024-11-20T16:53:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:26,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T16:53:26,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:26,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:26,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:26,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:26,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:26,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:26,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/4ec19189162e4fc4a3aec9512df41b46 is 50, key is test_row_0/A:col10/1732121606356/Put/seqid=0 2024-11-20T16:53:26,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742158_1334 (size=14741) 2024-11-20T16:53:26,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/4ec19189162e4fc4a3aec9512df41b46 2024-11-20T16:53:26,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/972e73b2b47446b9bcde025490f72375 is 50, key is test_row_0/B:col10/1732121606356/Put/seqid=0 2024-11-20T16:53:26,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742159_1335 (size=12301) 2024-11-20T16:53:26,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121666786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T16:53:26,796 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:26,796 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-20T16:53:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T16:53:26,797 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:26,798 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:26,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:26,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121666896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:26,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:26,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:26,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T16:53:26,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T16:53:26,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:26,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:26,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:26,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:26,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:27,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T16:53:27,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:27,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:27,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:27,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:27,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:27,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121667106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T16:53:27,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/972e73b2b47446b9bcde025490f72375 2024-11-20T16:53:27,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/c69a6799c4c54f41b5ca71b786b6d9cd is 50, key is test_row_0/C:col10/1732121606356/Put/seqid=0 2024-11-20T16:53:27,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742160_1336 (size=12301) 2024-11-20T16:53:27,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/c69a6799c4c54f41b5ca71b786b6d9cd 2024-11-20T16:53:27,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/4ec19189162e4fc4a3aec9512df41b46 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/4ec19189162e4fc4a3aec9512df41b46 2024-11-20T16:53:27,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/4ec19189162e4fc4a3aec9512df41b46, entries=200, sequenceid=338, filesize=14.4 K 2024-11-20T16:53:27,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/972e73b2b47446b9bcde025490f72375 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/972e73b2b47446b9bcde025490f72375 2024-11-20T16:53:27,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/972e73b2b47446b9bcde025490f72375, entries=150, sequenceid=338, filesize=12.0 K 2024-11-20T16:53:27,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/c69a6799c4c54f41b5ca71b786b6d9cd as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/c69a6799c4c54f41b5ca71b786b6d9cd 2024-11-20T16:53:27,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/c69a6799c4c54f41b5ca71b786b6d9cd, entries=150, sequenceid=338, filesize=12.0 K 2024-11-20T16:53:27,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T16:53:27,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:27,255 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-20T16:53:27,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:27,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:27,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:27,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:27,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e1410d95b5f65cde4303e83d1cb47b81 in 557ms, sequenceid=338, compaction requested=true 2024-11-20T16:53:27,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:27,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:53:27,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:27,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:53:27,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T16:53:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:27,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:53:27,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T16:53:27,256 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:27,256 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:27,258 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:27,258 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/A is initiating minor compaction (all files) 2024-11-20T16:53:27,258 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:27,258 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/A in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:27,258 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/B is initiating minor compaction (all files) 2024-11-20T16:53:27,258 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/B in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:27,258 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/2bb9ad17b55c4c1dbaca9aa78316b118, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/c82a7e7d93ac49fc8d74c94ee3b5548a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/4ec19189162e4fc4a3aec9512df41b46] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=39.0 K 2024-11-20T16:53:27,258 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/f87c648e4c3442069f0d8c40a63e66ee, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/710e79f9e4134ff894095587e5c3548a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/972e73b2b47446b9bcde025490f72375] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=36.6 K 2024-11-20T16:53:27,258 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bb9ad17b55c4c1dbaca9aa78316b118, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732121603879 2024-11-20T16:53:27,258 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting f87c648e4c3442069f0d8c40a63e66ee, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732121603879 2024-11-20T16:53:27,259 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 710e79f9e4134ff894095587e5c3548a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732121605174 2024-11-20T16:53:27,259 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting c82a7e7d93ac49fc8d74c94ee3b5548a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732121605174 2024-11-20T16:53:27,259 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 972e73b2b47446b9bcde025490f72375, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732121606356 2024-11-20T16:53:27,259 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ec19189162e4fc4a3aec9512df41b46, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732121606356 2024-11-20T16:53:27,269 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#B#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:27,269 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/09068cc7828046309916bb06be12fcb7 is 50, key is test_row_0/B:col10/1732121606356/Put/seqid=0 2024-11-20T16:53:27,274 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#A#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:27,274 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/5cf230d40e5e4a78a069b831cd9d44d2 is 50, key is test_row_0/A:col10/1732121606356/Put/seqid=0 2024-11-20T16:53:27,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742161_1337 (size=12949) 2024-11-20T16:53:27,330 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/09068cc7828046309916bb06be12fcb7 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/09068cc7828046309916bb06be12fcb7 2024-11-20T16:53:27,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742162_1338 (size=12949) 2024-11-20T16:53:27,341 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/B of e1410d95b5f65cde4303e83d1cb47b81 into 09068cc7828046309916bb06be12fcb7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:27,341 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:27,341 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/B, priority=13, startTime=1732121607256; duration=0sec 2024-11-20T16:53:27,341 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:27,341 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:B 2024-11-20T16:53:27,341 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:27,344 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:27,344 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/C is initiating minor compaction (all files) 2024-11-20T16:53:27,344 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/C in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:27,345 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/ecca7eb36bfb4da98878def401b9e414, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/60b762ec1c164114a70c8fd4c70f9fc9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/c69a6799c4c54f41b5ca71b786b6d9cd] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=36.5 K 2024-11-20T16:53:27,345 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting ecca7eb36bfb4da98878def401b9e414, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732121603879 2024-11-20T16:53:27,346 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 60b762ec1c164114a70c8fd4c70f9fc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732121605174 2024-11-20T16:53:27,346 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting c69a6799c4c54f41b5ca71b786b6d9cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732121606356 2024-11-20T16:53:27,347 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/5cf230d40e5e4a78a069b831cd9d44d2 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/5cf230d40e5e4a78a069b831cd9d44d2 2024-11-20T16:53:27,352 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/A of e1410d95b5f65cde4303e83d1cb47b81 into 5cf230d40e5e4a78a069b831cd9d44d2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:27,352 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:27,352 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/A, priority=13, startTime=1732121607256; duration=0sec 2024-11-20T16:53:27,352 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:27,352 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:A 2024-11-20T16:53:27,355 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#C#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:27,355 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/2d56b5ffc35a4629aec0d1b94bc7888a is 50, key is test_row_0/C:col10/1732121606356/Put/seqid=0 2024-11-20T16:53:27,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742163_1339 (size=12915) 2024-11-20T16:53:27,380 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/2d56b5ffc35a4629aec0d1b94bc7888a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/2d56b5ffc35a4629aec0d1b94bc7888a 2024-11-20T16:53:27,387 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/C of e1410d95b5f65cde4303e83d1cb47b81 into 2d56b5ffc35a4629aec0d1b94bc7888a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:27,387 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:27,387 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/C, priority=13, startTime=1732121607256; duration=0sec 2024-11-20T16:53:27,388 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:27,388 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:C 2024-11-20T16:53:27,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T16:53:27,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:27,408 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T16:53:27,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:27,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:27,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:27,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:27,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:27,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:27,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/e9c052ba4be34115a5dc821c89d4489a is 50, key is test_row_0/A:col10/1732121606753/Put/seqid=0 2024-11-20T16:53:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:27,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T16:53:27,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742164_1340 (size=12301) 2024-11-20T16:53:27,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121667464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:27,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121667576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:27,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:27,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:27,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:27,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121667786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:27,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:27,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:27,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:27,849 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/e9c052ba4be34115a5dc821c89d4489a 2024-11-20T16:53:27,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/6c3b6d571fda4e0883e34a310f995091 is 50, key is test_row_0/B:col10/1732121606753/Put/seqid=0 2024-11-20T16:53:27,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742165_1341 (size=12301) 2024-11-20T16:53:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T16:53:28,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121668096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:28,262 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/6c3b6d571fda4e0883e34a310f995091 2024-11-20T16:53:28,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/6e9eda8d71c64675b8ea7c6be1e34b78 is 50, key is test_row_0/C:col10/1732121606753/Put/seqid=0 2024-11-20T16:53:28,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742166_1342 (size=12301) 2024-11-20T16:53:28,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121668606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:28,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:28,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:28,674 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/6e9eda8d71c64675b8ea7c6be1e34b78 2024-11-20T16:53:28,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/e9c052ba4be34115a5dc821c89d4489a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/e9c052ba4be34115a5dc821c89d4489a 2024-11-20T16:53:28,681 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/e9c052ba4be34115a5dc821c89d4489a, entries=150, sequenceid=360, filesize=12.0 K 2024-11-20T16:53:28,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/6c3b6d571fda4e0883e34a310f995091 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/6c3b6d571fda4e0883e34a310f995091 2024-11-20T16:53:28,697 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/6c3b6d571fda4e0883e34a310f995091, entries=150, sequenceid=360, filesize=12.0 K 2024-11-20T16:53:28,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/6e9eda8d71c64675b8ea7c6be1e34b78 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/6e9eda8d71c64675b8ea7c6be1e34b78 2024-11-20T16:53:28,702 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/6e9eda8d71c64675b8ea7c6be1e34b78, entries=150, sequenceid=360, filesize=12.0 K 2024-11-20T16:53:28,703 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e1410d95b5f65cde4303e83d1cb47b81 in 1295ms, sequenceid=360, compaction requested=false 2024-11-20T16:53:28,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:28,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:28,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-20T16:53:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-20T16:53:28,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-20T16:53:28,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9070 sec 2024-11-20T16:53:28,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.9120 sec 2024-11-20T16:53:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T16:53:28,956 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:28,956 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-20T16:53:28,958 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T16:53:28,959 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:28,959 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T16:53:29,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-20T16:53:29,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:29,112 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T16:53:29,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:29,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:29,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:29,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:29,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:29,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:29,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/fd9af7bedf274dd2ae5077b7a8051e4c is 50, key is test_row_0/A:col10/1732121607458/Put/seqid=0 2024-11-20T16:53:29,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742167_1343 (size=12301) 2024-11-20T16:53:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T16:53:29,532 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/fd9af7bedf274dd2ae5077b7a8051e4c 2024-11-20T16:53:29,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/5680f25ee3694a31ba88c73698cd7aed is 50, key is test_row_0/B:col10/1732121607458/Put/seqid=0 2024-11-20T16:53:29,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742168_1344 (size=12301) 2024-11-20T16:53:29,542 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/5680f25ee3694a31ba88c73698cd7aed 2024-11-20T16:53:29,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/c3b725437836405e9ea71da329e1876e is 50, key is test_row_0/C:col10/1732121607458/Put/seqid=0 2024-11-20T16:53:29,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742169_1345 (size=12301) 2024-11-20T16:53:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T16:53:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:29,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:29,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121669692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:29,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121669806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:29,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:29,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:29,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:29,966 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/c3b725437836405e9ea71da329e1876e 2024-11-20T16:53:29,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/fd9af7bedf274dd2ae5077b7a8051e4c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/fd9af7bedf274dd2ae5077b7a8051e4c 2024-11-20T16:53:29,981 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/fd9af7bedf274dd2ae5077b7a8051e4c, entries=150, sequenceid=373, filesize=12.0 K 2024-11-20T16:53:29,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/5680f25ee3694a31ba88c73698cd7aed as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/5680f25ee3694a31ba88c73698cd7aed 2024-11-20T16:53:29,993 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/5680f25ee3694a31ba88c73698cd7aed, entries=150, sequenceid=373, filesize=12.0 K 2024-11-20T16:53:29,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/c3b725437836405e9ea71da329e1876e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/c3b725437836405e9ea71da329e1876e 2024-11-20T16:53:29,998 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/c3b725437836405e9ea71da329e1876e, entries=150, sequenceid=373, filesize=12.0 K 2024-11-20T16:53:29,999 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e1410d95b5f65cde4303e83d1cb47b81 in 886ms, sequenceid=373, compaction requested=true 2024-11-20T16:53:29,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:29,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:29,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-20T16:53:29,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-20T16:53:30,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-20T16:53:30,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0410 sec 2024-11-20T16:53:30,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.0470 sec 2024-11-20T16:53:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:30,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T16:53:30,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:30,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:30,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:30,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:30,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:30,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:30,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/4b2646bdc6dc401b86a96c63af5ceedf is 50, key is test_row_0/A:col10/1732121609668/Put/seqid=0 2024-11-20T16:53:30,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742170_1346 (size=14741) 2024-11-20T16:53:30,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121670083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T16:53:30,096 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:30,096 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:30,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-20T16:53:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-20T16:53:30,098 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:30,098 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:30,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:30,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121670196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:30,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:30,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:30,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-20T16:53:30,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-20T16:53:30,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:30,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:30,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:30,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:30,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:30,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-20T16:53:30,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:30,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:30,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:30,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:30,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:30,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121670406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:30,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-20T16:53:30,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/4b2646bdc6dc401b86a96c63af5ceedf 2024-11-20T16:53:30,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/c417b22b30624068aaddacd41d86fae2 is 50, key is test_row_0/B:col10/1732121609668/Put/seqid=0 2024-11-20T16:53:30,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742171_1347 (size=12301) 2024-11-20T16:53:30,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/c417b22b30624068aaddacd41d86fae2 2024-11-20T16:53:30,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/02f17e715008405cb1d4c766eb96ddd9 is 50, key is test_row_0/C:col10/1732121609668/Put/seqid=0 2024-11-20T16:53:30,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742172_1348 (size=12301) 2024-11-20T16:53:30,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/02f17e715008405cb1d4c766eb96ddd9 2024-11-20T16:53:30,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/4b2646bdc6dc401b86a96c63af5ceedf as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/4b2646bdc6dc401b86a96c63af5ceedf 2024-11-20T16:53:30,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/4b2646bdc6dc401b86a96c63af5ceedf, entries=200, sequenceid=393, filesize=14.4 K 2024-11-20T16:53:30,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/c417b22b30624068aaddacd41d86fae2 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/c417b22b30624068aaddacd41d86fae2 2024-11-20T16:53:30,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/c417b22b30624068aaddacd41d86fae2, entries=150, sequenceid=393, filesize=12.0 K 2024-11-20T16:53:30,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/02f17e715008405cb1d4c766eb96ddd9 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/02f17e715008405cb1d4c766eb96ddd9 2024-11-20T16:53:30,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/02f17e715008405cb1d4c766eb96ddd9, entries=150, sequenceid=393, filesize=12.0 K 2024-11-20T16:53:30,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e1410d95b5f65cde4303e83d1cb47b81 in 457ms, sequenceid=393, compaction requested=true 2024-11-20T16:53:30,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:30,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:53:30,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:30,479 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:53:30,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:53:30,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:30,479 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:53:30,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:53:30,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:30,480 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:53:30,480 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/A is initiating minor compaction (all files) 2024-11-20T16:53:30,480 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/A in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:30,480 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/5cf230d40e5e4a78a069b831cd9d44d2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/e9c052ba4be34115a5dc821c89d4489a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/fd9af7bedf274dd2ae5077b7a8051e4c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/4b2646bdc6dc401b86a96c63af5ceedf] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=51.1 K 2024-11-20T16:53:30,480 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:53:30,480 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/B is initiating minor compaction (all files) 2024-11-20T16:53:30,480 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/B in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:30,480 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/09068cc7828046309916bb06be12fcb7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/6c3b6d571fda4e0883e34a310f995091, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/5680f25ee3694a31ba88c73698cd7aed, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/c417b22b30624068aaddacd41d86fae2] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=48.7 K 2024-11-20T16:53:30,482 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cf230d40e5e4a78a069b831cd9d44d2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732121606356 2024-11-20T16:53:30,482 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 09068cc7828046309916bb06be12fcb7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732121606356 2024-11-20T16:53:30,482 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9c052ba4be34115a5dc821c89d4489a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732121606753 2024-11-20T16:53:30,483 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c3b6d571fda4e0883e34a310f995091, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732121606753 2024-11-20T16:53:30,483 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd9af7bedf274dd2ae5077b7a8051e4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732121607447 2024-11-20T16:53:30,483 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 5680f25ee3694a31ba88c73698cd7aed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732121607447 2024-11-20T16:53:30,483 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b2646bdc6dc401b86a96c63af5ceedf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732121609668 2024-11-20T16:53:30,483 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting c417b22b30624068aaddacd41d86fae2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732121609668 2024-11-20T16:53:30,494 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#A#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:30,495 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#B#compaction#296 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:30,495 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/b902ad478add4eca86c7d594d2eb7da1 is 50, key is test_row_0/A:col10/1732121609668/Put/seqid=0 2024-11-20T16:53:30,495 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/90fe5d0fe7bd4fff966249b12540f6c7 is 50, key is test_row_0/B:col10/1732121609668/Put/seqid=0 2024-11-20T16:53:30,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742173_1349 (size=13085) 2024-11-20T16:53:30,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742174_1350 (size=13085) 2024-11-20T16:53:30,514 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/90fe5d0fe7bd4fff966249b12540f6c7 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/90fe5d0fe7bd4fff966249b12540f6c7 2024-11-20T16:53:30,524 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/B of e1410d95b5f65cde4303e83d1cb47b81 into 90fe5d0fe7bd4fff966249b12540f6c7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:30,524 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:30,524 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/B, priority=12, startTime=1732121610479; duration=0sec 2024-11-20T16:53:30,525 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:30,525 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:B 2024-11-20T16:53:30,525 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:53:30,535 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:53:30,535 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/C is initiating minor compaction (all files) 2024-11-20T16:53:30,535 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/C in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:30,536 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/2d56b5ffc35a4629aec0d1b94bc7888a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/6e9eda8d71c64675b8ea7c6be1e34b78, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/c3b725437836405e9ea71da329e1876e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/02f17e715008405cb1d4c766eb96ddd9] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=48.7 K 2024-11-20T16:53:30,536 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d56b5ffc35a4629aec0d1b94bc7888a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732121606356 2024-11-20T16:53:30,536 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e9eda8d71c64675b8ea7c6be1e34b78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732121606753 2024-11-20T16:53:30,537 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting c3b725437836405e9ea71da329e1876e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732121607447 2024-11-20T16:53:30,537 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 02f17e715008405cb1d4c766eb96ddd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732121609668 2024-11-20T16:53:30,553 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#C#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:30,554 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/11020337cc0746139fe73594b079e05f is 50, key is test_row_0/C:col10/1732121609668/Put/seqid=0 2024-11-20T16:53:30,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742175_1351 (size=13051) 2024-11-20T16:53:30,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-20T16:53:30,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:30,566 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T16:53:30,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:30,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:30,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:30,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:30,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:30,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:30,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/560f2c299153451cb1ebe76f02416827 is 50, key is test_row_0/A:col10/1732121610081/Put/seqid=0 2024-11-20T16:53:30,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742176_1352 (size=12301) 2024-11-20T16:53:30,590 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/560f2c299153451cb1ebe76f02416827 2024-11-20T16:53:30,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/9722756323db4f73ac65e79ba3d539bd is 50, key is test_row_0/B:col10/1732121610081/Put/seqid=0 2024-11-20T16:53:30,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742177_1353 (size=12301) 2024-11-20T16:53:30,600 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/9722756323db4f73ac65e79ba3d539bd 2024-11-20T16:53:30,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/73657fe38bb84e9298ecc2128061e6ca is 50, key is test_row_0/C:col10/1732121610081/Put/seqid=0 2024-11-20T16:53:30,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742178_1354 (size=12301) 2024-11-20T16:53:30,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-20T16:53:30,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121670817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:30,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:30,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:30,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:30,905 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/b902ad478add4eca86c7d594d2eb7da1 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/b902ad478add4eca86c7d594d2eb7da1 2024-11-20T16:53:30,908 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/A of e1410d95b5f65cde4303e83d1cb47b81 into b902ad478add4eca86c7d594d2eb7da1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:30,908 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:30,909 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/A, priority=12, startTime=1732121610479; duration=0sec 2024-11-20T16:53:30,909 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:30,909 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:A 2024-11-20T16:53:30,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121670926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:30,967 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/11020337cc0746139fe73594b079e05f as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/11020337cc0746139fe73594b079e05f 2024-11-20T16:53:30,971 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/C of e1410d95b5f65cde4303e83d1cb47b81 into 11020337cc0746139fe73594b079e05f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:30,972 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:30,972 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/C, priority=12, startTime=1732121610479; duration=0sec 2024-11-20T16:53:30,972 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:30,972 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:C 2024-11-20T16:53:31,022 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/73657fe38bb84e9298ecc2128061e6ca 2024-11-20T16:53:31,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/560f2c299153451cb1ebe76f02416827 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/560f2c299153451cb1ebe76f02416827 2024-11-20T16:53:31,032 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/560f2c299153451cb1ebe76f02416827, entries=150, sequenceid=406, filesize=12.0 K 2024-11-20T16:53:31,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/9722756323db4f73ac65e79ba3d539bd as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/9722756323db4f73ac65e79ba3d539bd 2024-11-20T16:53:31,038 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/9722756323db4f73ac65e79ba3d539bd, entries=150, sequenceid=406, filesize=12.0 K 2024-11-20T16:53:31,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/73657fe38bb84e9298ecc2128061e6ca as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/73657fe38bb84e9298ecc2128061e6ca 2024-11-20T16:53:31,044 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/73657fe38bb84e9298ecc2128061e6ca, entries=150, sequenceid=406, filesize=12.0 K 2024-11-20T16:53:31,045 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e1410d95b5f65cde4303e83d1cb47b81 in 479ms, sequenceid=406, compaction requested=false 2024-11-20T16:53:31,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:31,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:31,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-20T16:53:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-20T16:53:31,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-20T16:53:31,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 950 msec 2024-11-20T16:53:31,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 953 msec 2024-11-20T16:53:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:31,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-20T16:53:31,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:31,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:31,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:31,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:31,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:31,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:31,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/6dc6a08c8b174d33b95627a9743e6652 is 50, key is test_row_0/A:col10/1732121611138/Put/seqid=0 2024-11-20T16:53:31,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742179_1355 (size=14741) 2024-11-20T16:53:31,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121671186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:31,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-20T16:53:31,246 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:31,246 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-20T16:53:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T16:53:31,249 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:31,250 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:31,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:31,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121671296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:31,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:31,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:31,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T16:53:31,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-20T16:53:31,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:31,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:31,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:31,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:31,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:31,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121671506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:31,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:31,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/6dc6a08c8b174d33b95627a9743e6652 2024-11-20T16:53:31,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-20T16:53:31,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:31,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:31,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:31,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:31,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:31,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/835778b8e19c4b82b5e90217ffa311f1 is 50, key is test_row_0/B:col10/1732121611138/Put/seqid=0 2024-11-20T16:53:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T16:53:31,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742180_1356 (size=12301) 2024-11-20T16:53:31,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-20T16:53:31,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:31,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:31,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:31,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:31,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:31,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:31,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121671816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:31,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:31,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:31,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:31,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-20T16:53:31,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:31,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:31,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:31,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:31,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:31,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:31,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T16:53:31,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/835778b8e19c4b82b5e90217ffa311f1 2024-11-20T16:53:31,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/57fa2b71db3a4878a7f056eb426d35eb is 50, key is test_row_0/C:col10/1732121611138/Put/seqid=0 2024-11-20T16:53:32,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742181_1357 (size=12301) 2024-11-20T16:53:32,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-20T16:53:32,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:32,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:32,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:32,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:32,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:32,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-20T16:53:32,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:32,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:32,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:32,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:32,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:32,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-20T16:53:32,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:32,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:32,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:32,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:32,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:32,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121672326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:32,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:32,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:32,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T16:53:32,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/57fa2b71db3a4878a7f056eb426d35eb 2024-11-20T16:53:32,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/6dc6a08c8b174d33b95627a9743e6652 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/6dc6a08c8b174d33b95627a9743e6652 2024-11-20T16:53:32,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/6dc6a08c8b174d33b95627a9743e6652, entries=200, sequenceid=429, filesize=14.4 K 2024-11-20T16:53:32,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/835778b8e19c4b82b5e90217ffa311f1 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/835778b8e19c4b82b5e90217ffa311f1 2024-11-20T16:53:32,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/835778b8e19c4b82b5e90217ffa311f1, entries=150, sequenceid=429, filesize=12.0 K 2024-11-20T16:53:32,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/57fa2b71db3a4878a7f056eb426d35eb as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/57fa2b71db3a4878a7f056eb426d35eb 2024-11-20T16:53:32,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/57fa2b71db3a4878a7f056eb426d35eb, entries=150, sequenceid=429, filesize=12.0 K 2024-11-20T16:53:32,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for e1410d95b5f65cde4303e83d1cb47b81 in 1285ms, sequenceid=429, compaction requested=true 2024-11-20T16:53:32,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:32,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:53:32,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:32,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:53:32,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:32,425 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:32,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:53:32,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:32,425 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:32,425 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:32,425 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/B is initiating minor compaction (all files) 2024-11-20T16:53:32,425 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:32,426 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/B in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:32,426 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/A is initiating minor compaction (all files) 2024-11-20T16:53:32,426 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/A in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:32,426 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/90fe5d0fe7bd4fff966249b12540f6c7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/9722756323db4f73ac65e79ba3d539bd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/835778b8e19c4b82b5e90217ffa311f1] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=36.8 K 2024-11-20T16:53:32,426 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/b902ad478add4eca86c7d594d2eb7da1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/560f2c299153451cb1ebe76f02416827, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/6dc6a08c8b174d33b95627a9743e6652] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=39.2 K 2024-11-20T16:53:32,426 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting b902ad478add4eca86c7d594d2eb7da1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732121609668 2024-11-20T16:53:32,426 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 90fe5d0fe7bd4fff966249b12540f6c7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732121609668 2024-11-20T16:53:32,426 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 9722756323db4f73ac65e79ba3d539bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732121610048 2024-11-20T16:53:32,427 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 560f2c299153451cb1ebe76f02416827, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732121610048 2024-11-20T16:53:32,427 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 835778b8e19c4b82b5e90217ffa311f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732121610790 2024-11-20T16:53:32,428 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dc6a08c8b174d33b95627a9743e6652, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732121610790 2024-11-20T16:53:32,440 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#B#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:32,440 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/e5d4c1440b274c12a43e18c9559c3772 is 50, key is test_row_0/B:col10/1732121611138/Put/seqid=0 2024-11-20T16:53:32,449 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#A#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:32,449 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/53ee6cb069324a71978de41bf36b730e is 50, key is test_row_0/A:col10/1732121611138/Put/seqid=0 2024-11-20T16:53:32,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-20T16:53:32,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:32,476 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T16:53:32,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:32,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:32,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:32,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:32,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:32,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:32,943 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_960049648_22 at /127.0.0.1:41620 [Receiving block BP-2108073788-172.17.0.2-1732121522165:blk_1073741832_1008] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 466ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/cluster_e1fea354-106b-cccb-72d5-7b2ff6b355d1/data/data2/, blockId=1073741832, seqno=1407 2024-11-20T16:53:32,943 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_960049648_22 at /127.0.0.1:35380 [Receiving block BP-2108073788-172.17.0.2-1732121522165:blk_1073742182_1358] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 476ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/cluster_e1fea354-106b-cccb-72d5-7b2ff6b355d1/data/data2/, blockId=1073742182, seqno=0 2024-11-20T16:53:32,944 INFO [AsyncFSWAL-0-hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625-prefix:b60eac9bda43,33929,1732121526384 {}] wal.AbstractFSWAL(1368): Slow sync cost: 466 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35935,DS-030cc8b6-5d9f-4830-b83f-d248e453ef59,DISK]] 2024-11-20T16:53:32,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742182_1358 (size=13187) 2024-11-20T16:53:32,950 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/e5d4c1440b274c12a43e18c9559c3772 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/e5d4c1440b274c12a43e18c9559c3772 2024-11-20T16:53:32,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742183_1359 (size=13187) 2024-11-20T16:53:32,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/3167a9a50d1c4c259609804b88f6e808 is 50, key is test_row_0/A:col10/1732121611175/Put/seqid=0 2024-11-20T16:53:32,955 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/B of e1410d95b5f65cde4303e83d1cb47b81 into e5d4c1440b274c12a43e18c9559c3772(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:32,955 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:32,955 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/B, priority=13, startTime=1732121612425; duration=0sec 2024-11-20T16:53:32,955 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:32,955 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:B 2024-11-20T16:53:32,955 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:32,962 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:32,962 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): e1410d95b5f65cde4303e83d1cb47b81/C is initiating minor compaction (all files) 2024-11-20T16:53:32,962 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1410d95b5f65cde4303e83d1cb47b81/C in TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:32,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742184_1360 (size=12301) 2024-11-20T16:53:32,962 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/11020337cc0746139fe73594b079e05f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/73657fe38bb84e9298ecc2128061e6ca, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/57fa2b71db3a4878a7f056eb426d35eb] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp, totalSize=36.8 K 2024-11-20T16:53:32,962 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/53ee6cb069324a71978de41bf36b730e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/53ee6cb069324a71978de41bf36b730e 2024-11-20T16:53:32,963 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 11020337cc0746139fe73594b079e05f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732121609668 2024-11-20T16:53:32,963 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 73657fe38bb84e9298ecc2128061e6ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732121610048 2024-11-20T16:53:32,964 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 57fa2b71db3a4878a7f056eb426d35eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732121610790 2024-11-20T16:53:32,969 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/A of e1410d95b5f65cde4303e83d1cb47b81 into 53ee6cb069324a71978de41bf36b730e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:32,969 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:32,969 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/A, priority=13, startTime=1732121612424; duration=0sec 2024-11-20T16:53:32,969 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:32,969 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:A 2024-11-20T16:53:32,972 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1410d95b5f65cde4303e83d1cb47b81#C#compaction#307 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:32,973 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/73f60ba552ea4e1a9cb537162f6cbfbf is 50, key is test_row_0/C:col10/1732121611138/Put/seqid=0 2024-11-20T16:53:32,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742185_1361 (size=13153) 2024-11-20T16:53:32,987 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/73f60ba552ea4e1a9cb537162f6cbfbf as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/73f60ba552ea4e1a9cb537162f6cbfbf 2024-11-20T16:53:32,993 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1410d95b5f65cde4303e83d1cb47b81/C of e1410d95b5f65cde4303e83d1cb47b81 into 73f60ba552ea4e1a9cb537162f6cbfbf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:32,993 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:32,993 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., storeName=e1410d95b5f65cde4303e83d1cb47b81/C, priority=13, startTime=1732121612425; duration=0sec 2024-11-20T16:53:32,993 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:32,993 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:C 2024-11-20T16:53:33,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:33,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. as already flushing 2024-11-20T16:53:33,363 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/3167a9a50d1c4c259609804b88f6e808 2024-11-20T16:53:33,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/827ba7e28c82477ab786749d3ddea050 is 50, key is test_row_0/B:col10/1732121611175/Put/seqid=0 2024-11-20T16:53:33,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742186_1362 (size=12301) 2024-11-20T16:53:33,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T16:53:33,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121613412, max timestamp=1732121613455Num uniques 859; 2024-11-20T16:53:33,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121613412, max timestamp=1732121613455Num uniques 859; 2024-11-20T16:53:33,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:53:33,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:53:33,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121613412, max timestamp=1732121613455Num uniques 859; 2024-11-20T16:53:33,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:53:33,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 345 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121673544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:33,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:33,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:33,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:33,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 346 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121673656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:33,787 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/827ba7e28c82477ab786749d3ddea050 2024-11-20T16:53:33,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/2ae9631c366747edb2b19814b8d26d7c is 50, key is test_row_0/C:col10/1732121611175/Put/seqid=0 2024-11-20T16:53:33,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742187_1363 (size=12301) 2024-11-20T16:53:33,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 347 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121673866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:34,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:34,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 348 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46056 deadline: 1732121674175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:34,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:34,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:34,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:53:34,198 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/2ae9631c366747edb2b19814b8d26d7c 2024-11-20T16:53:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/3167a9a50d1c4c259609804b88f6e808 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/3167a9a50d1c4c259609804b88f6e808 2024-11-20T16:53:34,205 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/3167a9a50d1c4c259609804b88f6e808, entries=150, sequenceid=440, filesize=12.0 K 2024-11-20T16:53:34,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/827ba7e28c82477ab786749d3ddea050 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/827ba7e28c82477ab786749d3ddea050 2024-11-20T16:53:34,209 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/827ba7e28c82477ab786749d3ddea050, entries=150, sequenceid=440, filesize=12.0 K 2024-11-20T16:53:34,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/2ae9631c366747edb2b19814b8d26d7c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/2ae9631c366747edb2b19814b8d26d7c 2024-11-20T16:53:34,214 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/2ae9631c366747edb2b19814b8d26d7c, entries=150, sequenceid=440, filesize=12.0 K 2024-11-20T16:53:34,215 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e1410d95b5f65cde4303e83d1cb47b81 in 1739ms, sequenceid=440, compaction requested=false 2024-11-20T16:53:34,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:34,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:34,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-20T16:53:34,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-20T16:53:34,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-20T16:53:34,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9660 sec 2024-11-20T16:53:34,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 2.9730 sec 2024-11-20T16:53:34,592 INFO [Thread-1294 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1294. 2024-11-20T16:53:34,592 DEBUG [Thread-1294 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:34,592 DEBUG [Thread-1294 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:34,592 DEBUG [Thread-1294 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:34,592 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:34,593 INFO [Thread-1288 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1288. 2024-11-20T16:53:34,593 DEBUG [Thread-1288 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:34,593 DEBUG [Thread-1288 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:34,593 DEBUG [Thread-1288 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:34,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:34,595 INFO [Thread-1296 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1296. 2024-11-20T16:53:34,595 DEBUG [Thread-1296 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:34,595 DEBUG [Thread-1296 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:34,595 DEBUG [Thread-1296 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:34,595 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:34,597 INFO [Thread-1292 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1292. 2024-11-20T16:53:34,597 DEBUG [Thread-1292 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:34,597 DEBUG [Thread-1292 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:34,597 INFO [Thread-1290 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1290. 2024-11-20T16:53:34,597 DEBUG [Thread-1292 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:34,597 DEBUG [Thread-1290 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:34,597 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:34,597 DEBUG [Thread-1290 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:34,597 DEBUG [Thread-1290 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:34,597 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:34,673 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T16:53:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:34,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-20T16:53:34,688 INFO [Thread-1283 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1283. 2024-11-20T16:53:34,688 DEBUG [Thread-1283 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:34,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:34,688 DEBUG [Thread-1283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:34,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:34,689 DEBUG [Thread-1283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:34,689 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:34,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:34,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:34,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:34,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:34,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/94be2f21af7b4dbaac5f43c2000aa3d9 is 50, key is test_row_0/A:col10/1732121613530/Put/seqid=0 2024-11-20T16:53:34,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742188_1364 (size=12301) 2024-11-20T16:53:35,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/94be2f21af7b4dbaac5f43c2000aa3d9 2024-11-20T16:53:35,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/d1ca2d5c9b814e188f8b05d32f25845e is 50, key is test_row_0/B:col10/1732121613530/Put/seqid=0 2024-11-20T16:53:35,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742189_1365 (size=12301) 2024-11-20T16:53:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T16:53:35,417 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:35,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/d1ca2d5c9b814e188f8b05d32f25845e 2024-11-20T16:53:35,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/098cb385aa314bd08ff6ed4d3df54a6e is 50, key is test_row_0/C:col10/1732121613530/Put/seqid=0 2024-11-20T16:53:35,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742190_1366 (size=12301) 2024-11-20T16:53:35,547 INFO [Thread-1285 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1285. 2024-11-20T16:53:35,547 DEBUG [Thread-1285 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:35,547 DEBUG [Thread-1285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:35,548 DEBUG [Thread-1285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:35,548 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:35,598 INFO [Thread-1277 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1277. 2024-11-20T16:53:35,598 DEBUG [Thread-1277 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:35,598 DEBUG [Thread-1277 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:35,598 DEBUG [Thread-1277 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:35,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:35,628 INFO [Thread-1281 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1281. 2024-11-20T16:53:35,628 DEBUG [Thread-1281 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:35,628 DEBUG [Thread-1281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:35,628 DEBUG [Thread-1281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:35,628 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:35,677 INFO [Thread-1279 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1279. 2024-11-20T16:53:35,677 DEBUG [Thread-1279 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:53:35,677 DEBUG [Thread-1279 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:35,677 DEBUG [Thread-1279 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:35,677 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:35,678 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T16:53:35,678 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-20T16:53:35,678 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-20T16:53:35,678 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-20T16:53:35,678 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 287 2024-11-20T16:53:35,678 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-20T16:53:35,678 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T16:53:35,678 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2621 2024-11-20T16:53:35,678 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2513 2024-11-20T16:53:35,678 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2599 2024-11-20T16:53:35,678 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2629 2024-11-20T16:53:35,678 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2528 2024-11-20T16:53:35,678 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T16:53:35,678 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T16:53:35,678 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T16:53:35,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:35,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:35,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:53:35,679 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T16:53:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T16:53:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-20T16:53:35,681 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121615681"}]},"ts":"1732121615681"} 2024-11-20T16:53:35,682 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T16:53:35,682 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T16:53:35,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T16:53:35,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1410d95b5f65cde4303e83d1cb47b81, UNASSIGN}] 2024-11-20T16:53:35,685 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1410d95b5f65cde4303e83d1cb47b81, UNASSIGN 2024-11-20T16:53:35,685 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=e1410d95b5f65cde4303e83d1cb47b81, regionState=CLOSING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:35,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1410d95b5f65cde4303e83d1cb47b81, UNASSIGN because future has completed 2024-11-20T16:53:35,686 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T16:53:35,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:53:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-20T16:53:35,838 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:35,838 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T16:53:35,839 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing e1410d95b5f65cde4303e83d1cb47b81, disabling compactions & flushes 2024-11-20T16:53:35,839 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:35,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/098cb385aa314bd08ff6ed4d3df54a6e 2024-11-20T16:53:35,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/94be2f21af7b4dbaac5f43c2000aa3d9 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/94be2f21af7b4dbaac5f43c2000aa3d9 2024-11-20T16:53:35,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/94be2f21af7b4dbaac5f43c2000aa3d9, entries=150, sequenceid=480, filesize=12.0 K 2024-11-20T16:53:35,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/d1ca2d5c9b814e188f8b05d32f25845e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/d1ca2d5c9b814e188f8b05d32f25845e 2024-11-20T16:53:35,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/d1ca2d5c9b814e188f8b05d32f25845e, entries=150, sequenceid=480, filesize=12.0 K 2024-11-20T16:53:35,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/098cb385aa314bd08ff6ed4d3df54a6e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/098cb385aa314bd08ff6ed4d3df54a6e 2024-11-20T16:53:35,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/098cb385aa314bd08ff6ed4d3df54a6e, entries=150, sequenceid=480, filesize=12.0 K 2024-11-20T16:53:35,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=26.84 KB/27480 for e1410d95b5f65cde4303e83d1cb47b81 in 1252ms, sequenceid=480, compaction requested=true 2024-11-20T16:53:35,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1410d95b5f65cde4303e83d1cb47b81: 2024-11-20T16:53:35,940 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:35,940 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:35,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:53:35,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:35,940 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. because compaction request was cancelled 2024-11-20T16:53:35,940 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. after waiting 0 ms 2024-11-20T16:53:35,940 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:35,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:53:35,940 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:A 2024-11-20T16:53:35,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:35,940 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. because compaction request was cancelled 2024-11-20T16:53:35,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1410d95b5f65cde4303e83d1cb47b81:C, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:53:35,940 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:B 2024-11-20T16:53:35,940 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(2902): Flushing e1410d95b5f65cde4303e83d1cb47b81 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-20T16:53:35,940 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. because compaction request was cancelled 2024-11-20T16:53:35,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:35,940 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1410d95b5f65cde4303e83d1cb47b81:C 2024-11-20T16:53:35,940 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=A 2024-11-20T16:53:35,941 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:35,941 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=B 2024-11-20T16:53:35,941 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:35,941 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1410d95b5f65cde4303e83d1cb47b81, store=C 2024-11-20T16:53:35,941 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:35,943 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/a25830423eb24f70b5522a89cb3612a3 is 50, key is test_row_0/A:col10/1732121615676/Put/seqid=0 2024-11-20T16:53:35,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742191_1367 (size=9857) 2024-11-20T16:53:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-20T16:53:36,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-20T16:53:36,347 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/a25830423eb24f70b5522a89cb3612a3 2024-11-20T16:53:36,355 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/ef1cecd43fc8469f892f7b3987a1cc88 is 50, key is test_row_0/B:col10/1732121615676/Put/seqid=0 2024-11-20T16:53:36,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742192_1368 (size=9857) 2024-11-20T16:53:36,759 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/ef1cecd43fc8469f892f7b3987a1cc88 2024-11-20T16:53:36,765 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/e43a1efc277c40f292d295eff9dacfdc is 50, key is test_row_0/C:col10/1732121615676/Put/seqid=0 2024-11-20T16:53:36,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742193_1369 (size=9857) 2024-11-20T16:53:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-20T16:53:37,169 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/e43a1efc277c40f292d295eff9dacfdc 2024-11-20T16:53:37,172 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/A/a25830423eb24f70b5522a89cb3612a3 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/a25830423eb24f70b5522a89cb3612a3 2024-11-20T16:53:37,176 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/a25830423eb24f70b5522a89cb3612a3, entries=100, sequenceid=487, filesize=9.6 K 2024-11-20T16:53:37,177 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/B/ef1cecd43fc8469f892f7b3987a1cc88 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/ef1cecd43fc8469f892f7b3987a1cc88 2024-11-20T16:53:37,180 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/ef1cecd43fc8469f892f7b3987a1cc88, entries=100, sequenceid=487, filesize=9.6 K 2024-11-20T16:53:37,181 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/.tmp/C/e43a1efc277c40f292d295eff9dacfdc as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/e43a1efc277c40f292d295eff9dacfdc 2024-11-20T16:53:37,185 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/e43a1efc277c40f292d295eff9dacfdc, entries=100, sequenceid=487, filesize=9.6 K 2024-11-20T16:53:37,185 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e1410d95b5f65cde4303e83d1cb47b81 in 1245ms, sequenceid=487, compaction requested=true 2024-11-20T16:53:37,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/fa2a0e8e70aa40a1a024d6eaf7079c5a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/7c6ab23c4b7943508516703f4181559e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/ce8c5af24dbb4a1294a695091f2b7e4a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/9a8c431930644aef9902c6ae0d74bc52, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/49c3e13ac8a54fa4a2a51d21b55b0b5e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/f773a7218dcd4fb8acbe8b6a045f84bd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/0e4b9bb30a3a4814a308d768f5d2c611, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/5cc7481fe206407bba40aea6383d59d5, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/badfa826877c4706ac7cfd6d12dc17ee, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/2b767b5c161d48338bdf6edbc3e16162, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/409c36ec1e394e44b9ca6b499305a08e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/d35d13bdf5ca4287afd6eef95c160c3b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/42719accf1b241548ea76dbd8369693e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/3b004acbf46346e1b115701e88b0702a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/1ecb8b94afe14833a3f20004d6e232a8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/2b077852616a4cf59af01a61050f01ec, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/2bb9ad17b55c4c1dbaca9aa78316b118, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/c82a7e7d93ac49fc8d74c94ee3b5548a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/4ec19189162e4fc4a3aec9512df41b46, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/5cf230d40e5e4a78a069b831cd9d44d2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/e9c052ba4be34115a5dc821c89d4489a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/fd9af7bedf274dd2ae5077b7a8051e4c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/4b2646bdc6dc401b86a96c63af5ceedf, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/b902ad478add4eca86c7d594d2eb7da1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/560f2c299153451cb1ebe76f02416827, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/6dc6a08c8b174d33b95627a9743e6652] to archive 2024-11-20T16:53:37,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T16:53:37,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/fa2a0e8e70aa40a1a024d6eaf7079c5a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/fa2a0e8e70aa40a1a024d6eaf7079c5a 2024-11-20T16:53:37,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/7c6ab23c4b7943508516703f4181559e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/7c6ab23c4b7943508516703f4181559e 2024-11-20T16:53:37,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/ce8c5af24dbb4a1294a695091f2b7e4a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/ce8c5af24dbb4a1294a695091f2b7e4a 2024-11-20T16:53:37,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/9a8c431930644aef9902c6ae0d74bc52 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/9a8c431930644aef9902c6ae0d74bc52 2024-11-20T16:53:37,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/49c3e13ac8a54fa4a2a51d21b55b0b5e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/49c3e13ac8a54fa4a2a51d21b55b0b5e 2024-11-20T16:53:37,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/f773a7218dcd4fb8acbe8b6a045f84bd to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/f773a7218dcd4fb8acbe8b6a045f84bd 2024-11-20T16:53:37,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/0e4b9bb30a3a4814a308d768f5d2c611 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/0e4b9bb30a3a4814a308d768f5d2c611 2024-11-20T16:53:37,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/5cc7481fe206407bba40aea6383d59d5 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/5cc7481fe206407bba40aea6383d59d5 2024-11-20T16:53:37,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/badfa826877c4706ac7cfd6d12dc17ee to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/badfa826877c4706ac7cfd6d12dc17ee 2024-11-20T16:53:37,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/2b767b5c161d48338bdf6edbc3e16162 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/2b767b5c161d48338bdf6edbc3e16162 2024-11-20T16:53:37,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/409c36ec1e394e44b9ca6b499305a08e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/409c36ec1e394e44b9ca6b499305a08e 2024-11-20T16:53:37,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/d35d13bdf5ca4287afd6eef95c160c3b to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/d35d13bdf5ca4287afd6eef95c160c3b 2024-11-20T16:53:37,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/42719accf1b241548ea76dbd8369693e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/42719accf1b241548ea76dbd8369693e 2024-11-20T16:53:37,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/3b004acbf46346e1b115701e88b0702a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/3b004acbf46346e1b115701e88b0702a 2024-11-20T16:53:37,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/1ecb8b94afe14833a3f20004d6e232a8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/1ecb8b94afe14833a3f20004d6e232a8 2024-11-20T16:53:37,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/2b077852616a4cf59af01a61050f01ec to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/2b077852616a4cf59af01a61050f01ec 2024-11-20T16:53:37,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/2bb9ad17b55c4c1dbaca9aa78316b118 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/2bb9ad17b55c4c1dbaca9aa78316b118 2024-11-20T16:53:37,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/c82a7e7d93ac49fc8d74c94ee3b5548a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/c82a7e7d93ac49fc8d74c94ee3b5548a 2024-11-20T16:53:37,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/4ec19189162e4fc4a3aec9512df41b46 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/4ec19189162e4fc4a3aec9512df41b46 2024-11-20T16:53:37,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/5cf230d40e5e4a78a069b831cd9d44d2 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/5cf230d40e5e4a78a069b831cd9d44d2 2024-11-20T16:53:37,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/e9c052ba4be34115a5dc821c89d4489a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/e9c052ba4be34115a5dc821c89d4489a 2024-11-20T16:53:37,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/fd9af7bedf274dd2ae5077b7a8051e4c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/fd9af7bedf274dd2ae5077b7a8051e4c 2024-11-20T16:53:37,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/4b2646bdc6dc401b86a96c63af5ceedf to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/4b2646bdc6dc401b86a96c63af5ceedf 2024-11-20T16:53:37,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/b902ad478add4eca86c7d594d2eb7da1 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/b902ad478add4eca86c7d594d2eb7da1 2024-11-20T16:53:37,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/560f2c299153451cb1ebe76f02416827 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/560f2c299153451cb1ebe76f02416827 2024-11-20T16:53:37,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/6dc6a08c8b174d33b95627a9743e6652 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/6dc6a08c8b174d33b95627a9743e6652 2024-11-20T16:53:37,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/3d53b08e15824c7fa5495927a7015c45, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/306ffc6c4e9c4ad28736484a0ea665e2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/3b16e9aac3f0492da0738fcc4fe5f4ab, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/07b1719fe87d4415b4cc1b29add2835b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/46e0cb50c2d84123ae7ff3669d10f515, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/8b1a44d106664feba6388713481eebb5, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/6837ae935c1646b6b4e04a2c4cf2923a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/2b5a893f1eca43f9a376360fb607ace2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/f0cc980a68974b7fa7a83bb49dcd0b1e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/636e8a7aad324633bae88d30614a3692, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/19e489e0778243349917979508813ae4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/b81b0c132b41434a993d150bb11cabd1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/95ec9283d4eb4b669c67985b9a024bfb, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/1c24b6745f834e839046ee7823ff48b2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/d3057477cb5d4e0aa109af5cf62a7e58, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/f87c648e4c3442069f0d8c40a63e66ee, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/a0aafed670424e8e85534ad9edaefb59, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/710e79f9e4134ff894095587e5c3548a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/09068cc7828046309916bb06be12fcb7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/972e73b2b47446b9bcde025490f72375, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/6c3b6d571fda4e0883e34a310f995091, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/5680f25ee3694a31ba88c73698cd7aed, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/90fe5d0fe7bd4fff966249b12540f6c7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/c417b22b30624068aaddacd41d86fae2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/9722756323db4f73ac65e79ba3d539bd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/835778b8e19c4b82b5e90217ffa311f1] to archive 2024-11-20T16:53:37,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T16:53:37,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/3d53b08e15824c7fa5495927a7015c45 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/3d53b08e15824c7fa5495927a7015c45 2024-11-20T16:53:37,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/306ffc6c4e9c4ad28736484a0ea665e2 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/306ffc6c4e9c4ad28736484a0ea665e2 2024-11-20T16:53:37,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/3b16e9aac3f0492da0738fcc4fe5f4ab to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/3b16e9aac3f0492da0738fcc4fe5f4ab 2024-11-20T16:53:37,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/07b1719fe87d4415b4cc1b29add2835b to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/07b1719fe87d4415b4cc1b29add2835b 2024-11-20T16:53:37,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/46e0cb50c2d84123ae7ff3669d10f515 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/46e0cb50c2d84123ae7ff3669d10f515 2024-11-20T16:53:37,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/8b1a44d106664feba6388713481eebb5 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/8b1a44d106664feba6388713481eebb5 2024-11-20T16:53:37,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/6837ae935c1646b6b4e04a2c4cf2923a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/6837ae935c1646b6b4e04a2c4cf2923a 2024-11-20T16:53:37,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/2b5a893f1eca43f9a376360fb607ace2 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/2b5a893f1eca43f9a376360fb607ace2 2024-11-20T16:53:37,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/f0cc980a68974b7fa7a83bb49dcd0b1e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/f0cc980a68974b7fa7a83bb49dcd0b1e 2024-11-20T16:53:37,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/636e8a7aad324633bae88d30614a3692 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/636e8a7aad324633bae88d30614a3692 2024-11-20T16:53:37,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/19e489e0778243349917979508813ae4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/19e489e0778243349917979508813ae4 2024-11-20T16:53:37,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/b81b0c132b41434a993d150bb11cabd1 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/b81b0c132b41434a993d150bb11cabd1 2024-11-20T16:53:37,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/95ec9283d4eb4b669c67985b9a024bfb to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/95ec9283d4eb4b669c67985b9a024bfb 2024-11-20T16:53:37,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/1c24b6745f834e839046ee7823ff48b2 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/1c24b6745f834e839046ee7823ff48b2 2024-11-20T16:53:37,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/d3057477cb5d4e0aa109af5cf62a7e58 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/d3057477cb5d4e0aa109af5cf62a7e58 2024-11-20T16:53:37,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/f87c648e4c3442069f0d8c40a63e66ee to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/f87c648e4c3442069f0d8c40a63e66ee 2024-11-20T16:53:37,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/a0aafed670424e8e85534ad9edaefb59 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/a0aafed670424e8e85534ad9edaefb59 2024-11-20T16:53:37,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/710e79f9e4134ff894095587e5c3548a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/710e79f9e4134ff894095587e5c3548a 2024-11-20T16:53:37,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/09068cc7828046309916bb06be12fcb7 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/09068cc7828046309916bb06be12fcb7 2024-11-20T16:53:37,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/972e73b2b47446b9bcde025490f72375 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/972e73b2b47446b9bcde025490f72375 2024-11-20T16:53:37,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/6c3b6d571fda4e0883e34a310f995091 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/6c3b6d571fda4e0883e34a310f995091 2024-11-20T16:53:37,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/5680f25ee3694a31ba88c73698cd7aed to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/5680f25ee3694a31ba88c73698cd7aed 2024-11-20T16:53:37,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/90fe5d0fe7bd4fff966249b12540f6c7 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/90fe5d0fe7bd4fff966249b12540f6c7 2024-11-20T16:53:37,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/c417b22b30624068aaddacd41d86fae2 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/c417b22b30624068aaddacd41d86fae2 2024-11-20T16:53:37,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/9722756323db4f73ac65e79ba3d539bd to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/9722756323db4f73ac65e79ba3d539bd 2024-11-20T16:53:37,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/835778b8e19c4b82b5e90217ffa311f1 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/835778b8e19c4b82b5e90217ffa311f1 2024-11-20T16:53:37,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/03df61dfd79244448c1bdd4642cb145e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/b2bd8c3d505a4d50a2a005b64409607f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/36cff06bde1f433db6fee33f67b3dc21, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/8d674cc39f874d189398a6802ddd60da, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/897b6534afdf4022a57faf2f36b21382, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/70e9cd6f6635414d9ccb3ad30bf52c16, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/034cb435ceff4b6e84c326e739d49008, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/d863695033104cca8970150e3b08c44e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/b94412bde93e407eade0670582faf4fc, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/76fc5c2872d5412d86089bfa6d1feecb, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/0333f28624a3465886083b4d34516b99, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/e0f5e760736d4753a9f14df7e9adc910, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/eb883026bfdc47c1807f58db034233c1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/2761c1edb9a5481f89bd17d3741d3213, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/ecca7eb36bfb4da98878def401b9e414, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/7fa905805ef54af28f2926122c19f987, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/60b762ec1c164114a70c8fd4c70f9fc9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/2d56b5ffc35a4629aec0d1b94bc7888a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/c69a6799c4c54f41b5ca71b786b6d9cd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/6e9eda8d71c64675b8ea7c6be1e34b78, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/c3b725437836405e9ea71da329e1876e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/11020337cc0746139fe73594b079e05f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/02f17e715008405cb1d4c766eb96ddd9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/73657fe38bb84e9298ecc2128061e6ca, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/57fa2b71db3a4878a7f056eb426d35eb] to archive 2024-11-20T16:53:37,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T16:53:37,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/03df61dfd79244448c1bdd4642cb145e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/03df61dfd79244448c1bdd4642cb145e 2024-11-20T16:53:37,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/b2bd8c3d505a4d50a2a005b64409607f to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/b2bd8c3d505a4d50a2a005b64409607f 2024-11-20T16:53:37,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/36cff06bde1f433db6fee33f67b3dc21 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/36cff06bde1f433db6fee33f67b3dc21 2024-11-20T16:53:37,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/8d674cc39f874d189398a6802ddd60da to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/8d674cc39f874d189398a6802ddd60da 2024-11-20T16:53:37,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/897b6534afdf4022a57faf2f36b21382 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/897b6534afdf4022a57faf2f36b21382 2024-11-20T16:53:37,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/70e9cd6f6635414d9ccb3ad30bf52c16 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/70e9cd6f6635414d9ccb3ad30bf52c16 2024-11-20T16:53:37,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/034cb435ceff4b6e84c326e739d49008 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/034cb435ceff4b6e84c326e739d49008 2024-11-20T16:53:37,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/d863695033104cca8970150e3b08c44e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/d863695033104cca8970150e3b08c44e 2024-11-20T16:53:37,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/b94412bde93e407eade0670582faf4fc to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/b94412bde93e407eade0670582faf4fc 2024-11-20T16:53:37,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/76fc5c2872d5412d86089bfa6d1feecb to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/76fc5c2872d5412d86089bfa6d1feecb 2024-11-20T16:53:37,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/0333f28624a3465886083b4d34516b99 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/0333f28624a3465886083b4d34516b99 2024-11-20T16:53:37,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/e0f5e760736d4753a9f14df7e9adc910 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/e0f5e760736d4753a9f14df7e9adc910 2024-11-20T16:53:37,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/eb883026bfdc47c1807f58db034233c1 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/eb883026bfdc47c1807f58db034233c1 2024-11-20T16:53:37,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/2761c1edb9a5481f89bd17d3741d3213 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/2761c1edb9a5481f89bd17d3741d3213 2024-11-20T16:53:37,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/ecca7eb36bfb4da98878def401b9e414 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/ecca7eb36bfb4da98878def401b9e414 2024-11-20T16:53:37,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/7fa905805ef54af28f2926122c19f987 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/7fa905805ef54af28f2926122c19f987 2024-11-20T16:53:37,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/60b762ec1c164114a70c8fd4c70f9fc9 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/60b762ec1c164114a70c8fd4c70f9fc9 2024-11-20T16:53:37,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/2d56b5ffc35a4629aec0d1b94bc7888a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/2d56b5ffc35a4629aec0d1b94bc7888a 2024-11-20T16:53:37,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/c69a6799c4c54f41b5ca71b786b6d9cd to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/c69a6799c4c54f41b5ca71b786b6d9cd 2024-11-20T16:53:37,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/6e9eda8d71c64675b8ea7c6be1e34b78 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/6e9eda8d71c64675b8ea7c6be1e34b78 2024-11-20T16:53:37,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/c3b725437836405e9ea71da329e1876e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/c3b725437836405e9ea71da329e1876e 2024-11-20T16:53:37,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/11020337cc0746139fe73594b079e05f to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/11020337cc0746139fe73594b079e05f 2024-11-20T16:53:37,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/02f17e715008405cb1d4c766eb96ddd9 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/02f17e715008405cb1d4c766eb96ddd9 2024-11-20T16:53:37,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/73657fe38bb84e9298ecc2128061e6ca to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/73657fe38bb84e9298ecc2128061e6ca 2024-11-20T16:53:37,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/57fa2b71db3a4878a7f056eb426d35eb to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/57fa2b71db3a4878a7f056eb426d35eb 2024-11-20T16:53:37,272 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/recovered.edits/490.seqid, newMaxSeqId=490, maxSeqId=1 2024-11-20T16:53:37,272 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. 2024-11-20T16:53:37,272 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for e1410d95b5f65cde4303e83d1cb47b81: Waiting for close lock at 1732121615839Running coprocessor pre-close hooks at 1732121615839Disabling compacts and flushes for region at 1732121615839Disabling writes for close at 1732121615940 (+101 ms)Obtaining lock to block concurrent updates at 1732121615940Preparing flush snapshotting stores in e1410d95b5f65cde4303e83d1cb47b81 at 1732121615940Finished memstore snapshotting TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732121615941 (+1 ms)Flushing stores of TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81. at 1732121615941Flushing e1410d95b5f65cde4303e83d1cb47b81/A: creating writer at 1732121615941Flushing e1410d95b5f65cde4303e83d1cb47b81/A: appending metadata at 1732121615943 (+2 ms)Flushing e1410d95b5f65cde4303e83d1cb47b81/A: closing flushed file at 1732121615943Flushing e1410d95b5f65cde4303e83d1cb47b81/B: creating writer at 1732121616350 (+407 ms)Flushing e1410d95b5f65cde4303e83d1cb47b81/B: appending metadata at 1732121616355 (+5 ms)Flushing e1410d95b5f65cde4303e83d1cb47b81/B: closing flushed file at 1732121616355Flushing e1410d95b5f65cde4303e83d1cb47b81/C: creating writer at 1732121616762 (+407 ms)Flushing e1410d95b5f65cde4303e83d1cb47b81/C: appending metadata at 1732121616764 (+2 ms)Flushing e1410d95b5f65cde4303e83d1cb47b81/C: closing flushed file at 1732121616764Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@241220ff: reopening flushed file at 1732121617172 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c19122e: reopening flushed file at 1732121617176 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2137c643: reopening flushed file at 1732121617180 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e1410d95b5f65cde4303e83d1cb47b81 in 1245ms, sequenceid=487, compaction requested=true at 1732121617185 (+5 ms)Writing region close event to WAL at 1732121617269 (+84 ms)Running coprocessor post-close hooks at 1732121617272 (+3 ms)Closed at 1732121617272 2024-11-20T16:53:37,275 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:37,275 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=e1410d95b5f65cde4303e83d1cb47b81, regionState=CLOSED 2024-11-20T16:53:37,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 because future has completed 2024-11-20T16:53:37,280 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-20T16:53:37,280 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure e1410d95b5f65cde4303e83d1cb47b81, server=b60eac9bda43,33929,1732121526384 in 1.5930 sec 2024-11-20T16:53:37,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-20T16:53:37,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1410d95b5f65cde4303e83d1cb47b81, UNASSIGN in 1.5960 sec 2024-11-20T16:53:37,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-20T16:53:37,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5990 sec 2024-11-20T16:53:37,285 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121617285"}]},"ts":"1732121617285"} 2024-11-20T16:53:37,287 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T16:53:37,287 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T16:53:37,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6090 sec 2024-11-20T16:53:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-20T16:53:37,837 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:37,837 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T16:53:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:53:37,839 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:53:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-20T16:53:37,840 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=96, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:53:37,842 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:37,844 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A, FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B, FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C, FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/recovered.edits] 2024-11-20T16:53:37,846 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/3167a9a50d1c4c259609804b88f6e808 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/3167a9a50d1c4c259609804b88f6e808 2024-11-20T16:53:37,847 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/53ee6cb069324a71978de41bf36b730e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/53ee6cb069324a71978de41bf36b730e 2024-11-20T16:53:37,848 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/94be2f21af7b4dbaac5f43c2000aa3d9 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/94be2f21af7b4dbaac5f43c2000aa3d9 2024-11-20T16:53:37,850 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/a25830423eb24f70b5522a89cb3612a3 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/A/a25830423eb24f70b5522a89cb3612a3 2024-11-20T16:53:37,852 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/827ba7e28c82477ab786749d3ddea050 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/827ba7e28c82477ab786749d3ddea050 2024-11-20T16:53:37,853 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/d1ca2d5c9b814e188f8b05d32f25845e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/d1ca2d5c9b814e188f8b05d32f25845e 2024-11-20T16:53:37,854 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/e5d4c1440b274c12a43e18c9559c3772 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/e5d4c1440b274c12a43e18c9559c3772 2024-11-20T16:53:37,855 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/ef1cecd43fc8469f892f7b3987a1cc88 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/B/ef1cecd43fc8469f892f7b3987a1cc88 2024-11-20T16:53:37,857 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/098cb385aa314bd08ff6ed4d3df54a6e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/098cb385aa314bd08ff6ed4d3df54a6e 2024-11-20T16:53:37,858 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/2ae9631c366747edb2b19814b8d26d7c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/2ae9631c366747edb2b19814b8d26d7c 2024-11-20T16:53:37,859 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/73f60ba552ea4e1a9cb537162f6cbfbf to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/73f60ba552ea4e1a9cb537162f6cbfbf 2024-11-20T16:53:37,860 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/e43a1efc277c40f292d295eff9dacfdc to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/C/e43a1efc277c40f292d295eff9dacfdc 2024-11-20T16:53:37,864 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/recovered.edits/490.seqid to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81/recovered.edits/490.seqid 2024-11-20T16:53:37,865 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/e1410d95b5f65cde4303e83d1cb47b81 2024-11-20T16:53:37,865 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T16:53:37,867 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=96, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:53:37,871 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T16:53:37,873 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T16:53:37,874 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=96, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:53:37,874 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T16:53:37,875 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732121617874"}]},"ts":"9223372036854775807"} 2024-11-20T16:53:37,877 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T16:53:37,877 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e1410d95b5f65cde4303e83d1cb47b81, NAME => 'TestAcidGuarantees,,1732121592372.e1410d95b5f65cde4303e83d1cb47b81.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T16:53:37,877 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T16:53:37,878 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732121617877"}]},"ts":"9223372036854775807"} 2024-11-20T16:53:37,880 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T16:53:37,881 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=96, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:53:37,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-20T16:53:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-20T16:53:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T16:53:37,947 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:37,954 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-48111169_22 at /127.0.0.1:33770 [Waiting for operation #396] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_960049648_22 at /127.0.0.1:50006 [Waiting for operation #165] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=729 (was 580) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5644 (was 6018) 2024-11-20T16:53:37,961 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=729, ProcessCount=11, AvailableMemoryMB=5644 2024-11-20T16:53:37,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T16:53:37,963 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T16:53:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T16:53:37,964 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T16:53:37,964 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:37,964 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 97 2024-11-20T16:53:37,965 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T16:53:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T16:53:37,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742194_1370 (size=1061) 2024-11-20T16:53:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T16:53:38,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T16:53:38,386 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625 2024-11-20T16:53:38,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742195_1371 (size=53) 2024-11-20T16:53:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T16:53:38,794 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:53:38,794 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5cb6af7207e610b3daed9e5466ca8411, disabling compactions & flushes 2024-11-20T16:53:38,794 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:38,794 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:38,794 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. after waiting 0 ms 2024-11-20T16:53:38,794 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:38,794 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:38,794 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5cb6af7207e610b3daed9e5466ca8411: Waiting for close lock at 1732121618794Disabling compacts and flushes for region at 1732121618794Disabling writes for close at 1732121618794Writing region close event to WAL at 1732121618794Closed at 1732121618794 2024-11-20T16:53:38,795 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T16:53:38,795 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732121618795"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732121618795"}]},"ts":"1732121618795"} 2024-11-20T16:53:38,797 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T16:53:38,797 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T16:53:38,798 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121618797"}]},"ts":"1732121618797"} 2024-11-20T16:53:38,799 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T16:53:38,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5cb6af7207e610b3daed9e5466ca8411, ASSIGN}] 2024-11-20T16:53:38,800 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5cb6af7207e610b3daed9e5466ca8411, ASSIGN 2024-11-20T16:53:38,801 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5cb6af7207e610b3daed9e5466ca8411, ASSIGN; state=OFFLINE, location=b60eac9bda43,33929,1732121526384; forceNewPlan=false, retain=false 2024-11-20T16:53:38,951 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=5cb6af7207e610b3daed9e5466ca8411, regionState=OPENING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:38,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5cb6af7207e610b3daed9e5466ca8411, ASSIGN because future has completed 2024-11-20T16:53:38,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:53:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T16:53:39,107 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:39,107 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7752): Opening region: {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} 2024-11-20T16:53:39,108 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:39,108 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:53:39,108 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7794): checking encryption for 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:39,108 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7797): checking classloading for 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:39,109 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:39,110 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:53:39,110 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cb6af7207e610b3daed9e5466ca8411 columnFamilyName A 2024-11-20T16:53:39,110 DEBUG [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:39,110 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] regionserver.HStore(327): Store=5cb6af7207e610b3daed9e5466ca8411/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:53:39,110 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:39,111 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:53:39,112 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cb6af7207e610b3daed9e5466ca8411 columnFamilyName B 2024-11-20T16:53:39,112 DEBUG [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:39,112 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] regionserver.HStore(327): Store=5cb6af7207e610b3daed9e5466ca8411/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:53:39,112 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:39,113 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:53:39,113 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cb6af7207e610b3daed9e5466ca8411 columnFamilyName C 2024-11-20T16:53:39,113 DEBUG [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:39,113 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] regionserver.HStore(327): Store=5cb6af7207e610b3daed9e5466ca8411/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:53:39,113 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:39,113 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1038): replaying wal for 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:39,114 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:39,114 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:39,115 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1048): stopping wal replay for 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:39,115 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1060): Cleaning up temporary data for 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:39,115 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T16:53:39,116 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1093): writing seq id for 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:39,117 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T16:53:39,118 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1114): Opened 5cb6af7207e610b3daed9e5466ca8411; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73180920, jitterRate=0.09048068523406982}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T16:53:39,118 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:39,118 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1006): Region open journal for 5cb6af7207e610b3daed9e5466ca8411: Running coprocessor pre-open hook at 1732121619108Writing region info on filesystem at 1732121619108Initializing all the Stores at 1732121619108Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121619108Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121619109 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121619109Cleaning up temporary data from old regions at 1732121619115 (+6 ms)Running coprocessor post-open hooks at 1732121619118 (+3 ms)Region opened successfully at 1732121619118 2024-11-20T16:53:39,119 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., pid=99, masterSystemTime=1732121619105 2024-11-20T16:53:39,121 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:39,121 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:39,121 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=5cb6af7207e610b3daed9e5466ca8411, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:39,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 because future has completed 2024-11-20T16:53:39,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-20T16:53:39,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 in 170 msec 2024-11-20T16:53:39,126 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-20T16:53:39,126 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5cb6af7207e610b3daed9e5466ca8411, ASSIGN in 326 msec 2024-11-20T16:53:39,127 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T16:53:39,127 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121619127"}]},"ts":"1732121619127"} 2024-11-20T16:53:39,128 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T16:53:39,128 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T16:53:39,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-20T16:53:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T16:53:40,116 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:40,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300a5f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,118 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:40,118 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:40,119 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d66eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:40,121 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cb70289, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,122 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:40,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:40,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:40,125 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:40,126 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T16:53:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b60eac9bda43,33377,1732121525625 2024-11-20T16:53:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@48f3febb 2024-11-20T16:53:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T16:53:40,128 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T16:53:40,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T16:53:40,130 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T16:53:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T16:53:40,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742196_1372 (size=1097) 2024-11-20T16:53:40,149 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-20T16:53:40,149 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-20T16:53:40,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T16:53:40,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5cb6af7207e610b3daed9e5466ca8411, REOPEN/MOVE}] 2024-11-20T16:53:40,156 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5cb6af7207e610b3daed9e5466ca8411, REOPEN/MOVE 2024-11-20T16:53:40,157 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=5cb6af7207e610b3daed9e5466ca8411, regionState=CLOSING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:40,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5cb6af7207e610b3daed9e5466ca8411, REOPEN/MOVE because future has completed 2024-11-20T16:53:40,159 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T16:53:40,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:53:40,312 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(122): Close 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:40,312 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T16:53:40,312 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1722): Closing 5cb6af7207e610b3daed9e5466ca8411, disabling compactions & flushes 2024-11-20T16:53:40,312 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:40,312 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:40,312 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. after waiting 0 ms 2024-11-20T16:53:40,312 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:40,315 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T16:53:40,316 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:40,316 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1676): Region close journal for 5cb6af7207e610b3daed9e5466ca8411: Waiting for close lock at 1732121620312Running coprocessor pre-close hooks at 1732121620312Disabling compacts and flushes for region at 1732121620312Disabling writes for close at 1732121620312Writing region close event to WAL at 1732121620312Running coprocessor post-close hooks at 1732121620316 (+4 ms)Closed at 1732121620316 2024-11-20T16:53:40,316 WARN [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegionServer(3299): Not adding moved region record: 5cb6af7207e610b3daed9e5466ca8411 to self. 2024-11-20T16:53:40,319 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(157): Closed 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:40,319 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=5cb6af7207e610b3daed9e5466ca8411, regionState=CLOSED 2024-11-20T16:53:40,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 because future has completed 2024-11-20T16:53:40,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-20T16:53:40,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; CloseRegionProcedure 5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 in 163 msec 2024-11-20T16:53:40,325 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5cb6af7207e610b3daed9e5466ca8411, REOPEN/MOVE; state=CLOSED, location=b60eac9bda43,33929,1732121526384; forceNewPlan=false, retain=true 2024-11-20T16:53:40,476 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=5cb6af7207e610b3daed9e5466ca8411, regionState=OPENING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:40,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5cb6af7207e610b3daed9e5466ca8411, REOPEN/MOVE because future has completed 2024-11-20T16:53:40,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:53:40,633 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:40,633 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7752): Opening region: {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} 2024-11-20T16:53:40,633 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:40,633 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:53:40,633 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7794): checking encryption for 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:40,633 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7797): checking classloading for 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:40,634 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:40,635 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:53:40,636 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cb6af7207e610b3daed9e5466ca8411 columnFamilyName A 2024-11-20T16:53:40,637 DEBUG [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:40,637 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] regionserver.HStore(327): Store=5cb6af7207e610b3daed9e5466ca8411/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:53:40,637 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:40,638 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:53:40,638 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cb6af7207e610b3daed9e5466ca8411 columnFamilyName B 2024-11-20T16:53:40,638 DEBUG [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:40,639 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] regionserver.HStore(327): Store=5cb6af7207e610b3daed9e5466ca8411/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:53:40,639 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:40,639 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:53:40,639 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cb6af7207e610b3daed9e5466ca8411 columnFamilyName C 2024-11-20T16:53:40,640 DEBUG [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:40,640 INFO [StoreOpener-5cb6af7207e610b3daed9e5466ca8411-1 {}] regionserver.HStore(327): Store=5cb6af7207e610b3daed9e5466ca8411/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:53:40,640 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:40,640 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1038): replaying wal for 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:40,641 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:40,641 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:40,642 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1048): stopping wal replay for 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:40,642 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1060): Cleaning up temporary data for 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:40,642 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T16:53:40,644 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1093): writing seq id for 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:40,644 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1114): Opened 5cb6af7207e610b3daed9e5466ca8411; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62875203, jitterRate=-0.06308646500110626}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T16:53:40,644 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:40,645 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1006): Region open journal for 5cb6af7207e610b3daed9e5466ca8411: Running coprocessor pre-open hook at 1732121620633Writing region info on filesystem at 1732121620633Initializing all the Stores at 1732121620634 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121620634Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121620634Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121620634Cleaning up temporary data from old regions at 1732121620642 (+8 ms)Running coprocessor post-open hooks at 1732121620644 (+2 ms)Region opened successfully at 1732121620645 (+1 ms) 2024-11-20T16:53:40,646 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., pid=104, masterSystemTime=1732121620630 2024-11-20T16:53:40,647 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:40,647 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:40,648 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=5cb6af7207e610b3daed9e5466ca8411, regionState=OPEN, openSeqNum=5, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:40,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 because future has completed 2024-11-20T16:53:40,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=102 2024-11-20T16:53:40,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=102, state=SUCCESS, hasLock=false; OpenRegionProcedure 5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 in 172 msec 2024-11-20T16:53:40,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-20T16:53:40,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5cb6af7207e610b3daed9e5466ca8411, REOPEN/MOVE in 497 msec 2024-11-20T16:53:40,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-20T16:53:40,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 502 msec 2024-11-20T16:53:40,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 526 msec 2024-11-20T16:53:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-11-20T16:53:40,660 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:40,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1774ef6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,661 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:40,661 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:40,662 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56c094db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:40,663 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:40,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@997d67b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:40,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:40,666 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@792bbdb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:40,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:40,667 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:40,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b6dfa19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:40,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:40,670 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:40,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:40,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:40,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@613ec64f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:40,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:40,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:40,671 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128ff1fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:40,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:40,673 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:40,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68a93c92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:40,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:40,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:40,675 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:40,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bb64462, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,683 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:40,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:40,684 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:40,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:40,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:40,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@375fbd60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:40,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:40,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:40,685 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21b00226, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:40,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:40,688 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e8703de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:40,689 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51efcc77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,691 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:40,691 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:40,691 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1751e615, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:40,693 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a441cd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:40,695 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:40,695 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:40,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:40,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:40,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bb6c7b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:40,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:40,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:40,697 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:40,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b09b24d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,704 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:40,704 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:40,705 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67b3fad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:40,708 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39886, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:40,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39f8e72c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:53:40,711 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:53:40,712 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:53:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:53:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:53:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@507d14e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:53:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:53:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:53:40,718 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:53:40,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a18c304, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:53:40,720 DEBUG [Thread-1681 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:40,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:40,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:53:40,735 DEBUG [Thread-1692 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:40,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:40,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:40,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:40,739 DEBUG [Thread-1684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:53:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:40,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:53:40,755 DEBUG [Thread-1686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:40,757 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:40,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T16:53:40,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:53:40,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:40,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:53:40,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:40,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:53:40,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:53:40,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:40,764 DEBUG [Thread-1690 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:40,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:40,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:40,769 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:40,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:53:40,778 DEBUG [Thread-1673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:40,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:40,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:40,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:53:40,806 DEBUG [Thread-1688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:40,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:40,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:53:40,839 DEBUG [Thread-1679 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:40,839 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-20T16:53:40,841 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:40,842 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:40,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:40,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:40,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T16:53:40,844 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:40,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:53:40,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112096dfb948420247a18b878b7fb6fa7e58_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121620756/Put/seqid=0 2024-11-20T16:53:40,869 DEBUG [Thread-1675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:40,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:40,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:40,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:40,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:53:40,876 DEBUG [Thread-1677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:53:40,877 DEBUG [Thread-1677 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:53:40,877 DEBUG [Thread-1677 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:53:40,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:53:40,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:53:40,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121620866, max timestamp=1732121620916Num uniques 859; 2024-11-20T16:53:40,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121620866, max timestamp=1732121620916Num uniques 859; 2024-11-20T16:53:40,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:53:40,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:53:40,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121620866, max timestamp=1732121620916Num uniques 859; 2024-11-20T16:53:40,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:53:40,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742197_1373 (size=12154) 2024-11-20T16:53:40,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T16:53:40,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T16:53:40,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:40,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:40,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:40,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:40,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121681004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121681022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121681023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121681025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121681042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T16:53:41,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:41,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:41,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:41,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T16:53:41,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121681146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121681147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121681167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121681167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121681178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T16:53:41,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:41,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:41,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:41,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,322 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:41,338 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112096dfb948420247a18b878b7fb6fa7e58_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112096dfb948420247a18b878b7fb6fa7e58_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:41,343 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/0eb90718ee0f4de894bb8a470cf3a573, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:41,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/0eb90718ee0f4de894bb8a470cf3a573 is 175, key is test_row_0/A:col10/1732121620756/Put/seqid=0 2024-11-20T16:53:41,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742198_1374 (size=30955) 2024-11-20T16:53:41,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121681376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121681377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121681386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121681386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121681409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T16:53:41,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:41,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:41,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:41,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T16:53:41,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T16:53:41,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:41,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:41,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:41,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121681696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121681696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121681707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121681707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121681726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:41,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T16:53:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:41,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,788 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/0eb90718ee0f4de894bb8a470cf3a573 2024-11-20T16:53:41,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/b5e03b3e93fe443d8826ba67f5eed5f8 is 50, key is test_row_0/B:col10/1732121620756/Put/seqid=0 2024-11-20T16:53:41,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742199_1375 (size=12001) 2024-11-20T16:53:41,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/b5e03b3e93fe443d8826ba67f5eed5f8 2024-11-20T16:53:41,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/0d3dc930d74a463895e1f4e638445394 is 50, key is test_row_0/C:col10/1732121620756/Put/seqid=0 2024-11-20T16:53:41,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742200_1376 (size=12001) 2024-11-20T16:53:41,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/0d3dc930d74a463895e1f4e638445394 2024-11-20T16:53:41,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T16:53:41,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:41,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:41,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:41,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:41,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/0eb90718ee0f4de894bb8a470cf3a573 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0eb90718ee0f4de894bb8a470cf3a573 2024-11-20T16:53:41,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0eb90718ee0f4de894bb8a470cf3a573, entries=150, sequenceid=14, filesize=30.2 K 2024-11-20T16:53:41,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/b5e03b3e93fe443d8826ba67f5eed5f8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/b5e03b3e93fe443d8826ba67f5eed5f8 2024-11-20T16:53:41,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/b5e03b3e93fe443d8826ba67f5eed5f8, entries=150, sequenceid=14, filesize=11.7 K 2024-11-20T16:53:41,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/0d3dc930d74a463895e1f4e638445394 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/0d3dc930d74a463895e1f4e638445394 2024-11-20T16:53:41,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/0d3dc930d74a463895e1f4e638445394, entries=150, sequenceid=14, filesize=11.7 K 2024-11-20T16:53:41,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5cb6af7207e610b3daed9e5466ca8411 in 1182ms, sequenceid=14, compaction requested=false 2024-11-20T16:53:41,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T16:53:42,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T16:53:42,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:42,075 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-20T16:53:42,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:53:42,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:42,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:53:42,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:42,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:53:42,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:42,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c2cbf72c04c447e58fcb929b36b5cc94_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121621022/Put/seqid=0 2024-11-20T16:53:42,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742201_1377 (size=12154) 2024-11-20T16:53:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:42,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:42,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121682244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:42,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121682244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:42,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121682245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:42,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:42,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121682246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:42,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:42,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:42,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:42,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121682255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:42,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121682366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121682366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:42,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121682366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:42,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:42,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121682376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:42,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:42,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:42,515 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c2cbf72c04c447e58fcb929b36b5cc94_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c2cbf72c04c447e58fcb929b36b5cc94_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:42,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/c5215b51e51a40da8f0803585331daeb, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:42,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/c5215b51e51a40da8f0803585331daeb is 175, key is test_row_0/A:col10/1732121621022/Put/seqid=0 2024-11-20T16:53:42,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742202_1378 (size=30955) 2024-11-20T16:53:42,524 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/c5215b51e51a40da8f0803585331daeb 2024-11-20T16:53:42,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/5e466f09559e4953940c62aba1f649b9 is 50, key is test_row_0/B:col10/1732121621022/Put/seqid=0 2024-11-20T16:53:42,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742203_1379 (size=12001) 2024-11-20T16:53:42,571 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/5e466f09559e4953940c62aba1f649b9 2024-11-20T16:53:42,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121682586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121682587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121682587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121682587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:42,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/473b0fe3f6c94429b19a71a6d27d4503 is 50, key is test_row_0/C:col10/1732121621022/Put/seqid=0 2024-11-20T16:53:42,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:42,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742204_1380 (size=12001) 2024-11-20T16:53:42,612 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/473b0fe3f6c94429b19a71a6d27d4503 2024-11-20T16:53:42,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/c5215b51e51a40da8f0803585331daeb as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c5215b51e51a40da8f0803585331daeb 2024-11-20T16:53:42,626 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c5215b51e51a40da8f0803585331daeb, entries=150, sequenceid=50, filesize=30.2 K 2024-11-20T16:53:42,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/5e466f09559e4953940c62aba1f649b9 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/5e466f09559e4953940c62aba1f649b9 2024-11-20T16:53:42,631 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/5e466f09559e4953940c62aba1f649b9, entries=150, sequenceid=50, filesize=11.7 K 2024-11-20T16:53:42,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/473b0fe3f6c94429b19a71a6d27d4503 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/473b0fe3f6c94429b19a71a6d27d4503 2024-11-20T16:53:42,636 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/473b0fe3f6c94429b19a71a6d27d4503, entries=150, sequenceid=50, filesize=11.7 K 2024-11-20T16:53:42,636 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 5cb6af7207e610b3daed9e5466ca8411 in 561ms, sequenceid=50, compaction requested=false 2024-11-20T16:53:42,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:42,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:42,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-20T16:53:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-20T16:53:42,639 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-20T16:53:42,639 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7960 sec 2024-11-20T16:53:42,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.8010 sec 2024-11-20T16:53:42,817 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T16:53:42,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:42,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T16:53:42,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:53:42,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:42,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:53:42,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:42,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:53:42,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:42,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120df92cccc981b4c4badb0d10c124232fe_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121622251/Put/seqid=0 2024-11-20T16:53:42,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742205_1381 (size=17034) 2024-11-20T16:53:42,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121682957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:42,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121682959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121682959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:42,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121682959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T16:53:42,987 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:42,987 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-20T16:53:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T16:53:42,989 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:42,990 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:42,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T16:53:43,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121683086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121683087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121683087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121683087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T16:53:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:43,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:43,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:43,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121683275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,286 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 5cb6af7207e610b3daed9e5466ca8411 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T16:53:43,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:43,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:43,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:43,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:43,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:43,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T16:53:43,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121683306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121683306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121683306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121683307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,344 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:43,346 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120df92cccc981b4c4badb0d10c124232fe_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120df92cccc981b4c4badb0d10c124232fe_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:43,347 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/59e46a02378a4b9194cf71dc79b4ce33, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:43,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/59e46a02378a4b9194cf71dc79b4ce33 is 175, key is test_row_0/A:col10/1732121622251/Put/seqid=0 2024-11-20T16:53:43,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742206_1382 (size=48139) 2024-11-20T16:53:43,365 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/59e46a02378a4b9194cf71dc79b4ce33 2024-11-20T16:53:43,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/c5f5b9bfaaa94b7d8676611b5b58d1f8 is 50, key is test_row_0/B:col10/1732121622251/Put/seqid=0 2024-11-20T16:53:43,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742207_1383 (size=12001) 2024-11-20T16:53:43,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/c5f5b9bfaaa94b7d8676611b5b58d1f8 2024-11-20T16:53:43,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/6be37acc8e0e477481f9e2395b97a9e3 is 50, key is test_row_0/C:col10/1732121622251/Put/seqid=0 2024-11-20T16:53:43,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742208_1384 (size=12001) 2024-11-20T16:53:43,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/6be37acc8e0e477481f9e2395b97a9e3 2024-11-20T16:53:43,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/59e46a02378a4b9194cf71dc79b4ce33 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/59e46a02378a4b9194cf71dc79b4ce33 2024-11-20T16:53:43,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/59e46a02378a4b9194cf71dc79b4ce33, entries=250, sequenceid=62, filesize=47.0 K 2024-11-20T16:53:43,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/c5f5b9bfaaa94b7d8676611b5b58d1f8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/c5f5b9bfaaa94b7d8676611b5b58d1f8 2024-11-20T16:53:43,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/c5f5b9bfaaa94b7d8676611b5b58d1f8, entries=150, sequenceid=62, filesize=11.7 K 2024-11-20T16:53:43,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/6be37acc8e0e477481f9e2395b97a9e3 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/6be37acc8e0e477481f9e2395b97a9e3 2024-11-20T16:53:43,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/6be37acc8e0e477481f9e2395b97a9e3, entries=150, sequenceid=62, filesize=11.7 K 2024-11-20T16:53:43,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5cb6af7207e610b3daed9e5466ca8411 in 543ms, sequenceid=62, compaction requested=true 2024-11-20T16:53:43,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:43,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:53:43,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:43,445 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:43,445 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:43,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:53:43,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:43,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:53:43,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:43,446 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:43,446 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:43,446 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/B is initiating minor compaction (all files) 2024-11-20T16:53:43,446 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/A is initiating minor compaction (all files) 2024-11-20T16:53:43,446 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/A in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:43,446 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/B in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:43,447 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/b5e03b3e93fe443d8826ba67f5eed5f8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/5e466f09559e4953940c62aba1f649b9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/c5f5b9bfaaa94b7d8676611b5b58d1f8] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=35.2 K 2024-11-20T16:53:43,447 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0eb90718ee0f4de894bb8a470cf3a573, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c5215b51e51a40da8f0803585331daeb, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/59e46a02378a4b9194cf71dc79b4ce33] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=107.5 K 2024-11-20T16:53:43,447 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:43,447 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0eb90718ee0f4de894bb8a470cf3a573, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c5215b51e51a40da8f0803585331daeb, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/59e46a02378a4b9194cf71dc79b4ce33] 2024-11-20T16:53:43,447 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting b5e03b3e93fe443d8826ba67f5eed5f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732121620745 2024-11-20T16:53:43,447 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eb90718ee0f4de894bb8a470cf3a573, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732121620745 2024-11-20T16:53:43,447 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e466f09559e4953940c62aba1f649b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732121621021 2024-11-20T16:53:43,447 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5215b51e51a40da8f0803585331daeb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732121621021 2024-11-20T16:53:43,447 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting c5f5b9bfaaa94b7d8676611b5b58d1f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732121622251 2024-11-20T16:53:43,447 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59e46a02378a4b9194cf71dc79b4ce33, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732121622224 2024-11-20T16:53:43,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T16:53:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:43,453 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T16:53:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:53:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:53:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:53:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:43,466 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:43,468 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#B#compaction#325 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:43,468 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/f969fc95faa240fcbdb9f291b087d00d is 50, key is test_row_0/B:col10/1732121622251/Put/seqid=0 2024-11-20T16:53:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120606f6610d8384961bd6cc9848d8bdcae_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121622956/Put/seqid=0 2024-11-20T16:53:43,471 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411208f4447472299411a9dc640eed1c6d634_5cb6af7207e610b3daed9e5466ca8411 store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:43,474 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411208f4447472299411a9dc640eed1c6d634_5cb6af7207e610b3daed9e5466ca8411, store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:43,474 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208f4447472299411a9dc640eed1c6d634_5cb6af7207e610b3daed9e5466ca8411 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:43,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742209_1385 (size=12104) 2024-11-20T16:53:43,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742210_1386 (size=12154) 2024-11-20T16:53:43,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742211_1387 (size=4469) 2024-11-20T16:53:43,501 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#A#compaction#326 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:43,501 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/e6e6154c27b44820b3b7677e326bc8e0 is 175, key is test_row_0/A:col10/1732121622251/Put/seqid=0 2024-11-20T16:53:43,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742212_1388 (size=31058) 2024-11-20T16:53:43,517 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/e6e6154c27b44820b3b7677e326bc8e0 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/e6e6154c27b44820b3b7677e326bc8e0 2024-11-20T16:53:43,523 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/A of 5cb6af7207e610b3daed9e5466ca8411 into e6e6154c27b44820b3b7677e326bc8e0(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:43,523 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:43,523 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/A, priority=13, startTime=1732121623445; duration=0sec 2024-11-20T16:53:43,523 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:43,523 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:A 2024-11-20T16:53:43,524 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:43,525 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:43,525 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/C is initiating minor compaction (all files) 2024-11-20T16:53:43,525 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/C in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:43,525 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/0d3dc930d74a463895e1f4e638445394, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/473b0fe3f6c94429b19a71a6d27d4503, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/6be37acc8e0e477481f9e2395b97a9e3] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=35.2 K 2024-11-20T16:53:43,526 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d3dc930d74a463895e1f4e638445394, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732121620745 2024-11-20T16:53:43,526 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 473b0fe3f6c94429b19a71a6d27d4503, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732121621021 2024-11-20T16:53:43,526 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6be37acc8e0e477481f9e2395b97a9e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732121622251 2024-11-20T16:53:43,534 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#C#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:43,534 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/e95ab6e3f36e483a8b680d4238c4e070 is 50, key is test_row_0/C:col10/1732121622251/Put/seqid=0 2024-11-20T16:53:43,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742213_1389 (size=12104) 2024-11-20T16:53:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T16:53:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:43,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:43,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121683640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121683650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121683651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121683651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121683756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121683757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121683757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121683776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,883 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/f969fc95faa240fcbdb9f291b087d00d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/f969fc95faa240fcbdb9f291b087d00d 2024-11-20T16:53:43,886 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/B of 5cb6af7207e610b3daed9e5466ca8411 into f969fc95faa240fcbdb9f291b087d00d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:43,886 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:43,886 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/B, priority=13, startTime=1732121623445; duration=0sec 2024-11-20T16:53:43,887 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:43,887 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:B 2024-11-20T16:53:43,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:43,896 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120606f6610d8384961bd6cc9848d8bdcae_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120606f6610d8384961bd6cc9848d8bdcae_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:43,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/6139bcb0689d46bfa0dc1575d868d110, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:43,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/6139bcb0689d46bfa0dc1575d868d110 is 175, key is test_row_0/A:col10/1732121622956/Put/seqid=0 2024-11-20T16:53:43,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742214_1390 (size=30955) 2024-11-20T16:53:43,944 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/e95ab6e3f36e483a8b680d4238c4e070 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/e95ab6e3f36e483a8b680d4238c4e070 2024-11-20T16:53:43,958 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/C of 5cb6af7207e610b3daed9e5466ca8411 into e95ab6e3f36e483a8b680d4238c4e070(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:43,958 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:43,958 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/C, priority=13, startTime=1732121623445; duration=0sec 2024-11-20T16:53:43,958 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:43,958 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:C 2024-11-20T16:53:43,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121683966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121683966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121683975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:43,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121683986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T16:53:44,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121684278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:44,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:44,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121684286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:44,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:44,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:44,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:44,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121684296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:44,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:44,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:44,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:44,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121684297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:44,316 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/6139bcb0689d46bfa0dc1575d868d110 2024-11-20T16:53:44,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/a851663a9a4b4cc1850235b969292fe5 is 50, key is test_row_0/B:col10/1732121622956/Put/seqid=0 2024-11-20T16:53:44,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742215_1391 (size=12001) 2024-11-20T16:53:44,744 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/a851663a9a4b4cc1850235b969292fe5 2024-11-20T16:53:44,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/fc3993220a8c464ca4a1f7cea64c8aff is 50, key is test_row_0/C:col10/1732121622956/Put/seqid=0 2024-11-20T16:53:44,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:44,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121684796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:44,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121684806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:44,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121684806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:44,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742216_1392 (size=12001) 2024-11-20T16:53:44,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:44,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121684816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:44,827 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/fc3993220a8c464ca4a1f7cea64c8aff 2024-11-20T16:53:44,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:44,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:44,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:44,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/6139bcb0689d46bfa0dc1575d868d110 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/6139bcb0689d46bfa0dc1575d868d110 2024-11-20T16:53:44,843 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/6139bcb0689d46bfa0dc1575d868d110, entries=150, sequenceid=82, filesize=30.2 K 2024-11-20T16:53:44,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/a851663a9a4b4cc1850235b969292fe5 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/a851663a9a4b4cc1850235b969292fe5 2024-11-20T16:53:44,853 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/a851663a9a4b4cc1850235b969292fe5, entries=150, sequenceid=82, filesize=11.7 K 2024-11-20T16:53:44,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/fc3993220a8c464ca4a1f7cea64c8aff as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/fc3993220a8c464ca4a1f7cea64c8aff 2024-11-20T16:53:44,861 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/fc3993220a8c464ca4a1f7cea64c8aff, entries=150, sequenceid=82, filesize=11.7 K 2024-11-20T16:53:44,862 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5cb6af7207e610b3daed9e5466ca8411 in 1409ms, sequenceid=82, compaction requested=false 2024-11-20T16:53:44,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:44,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:44,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-20T16:53:44,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-20T16:53:44,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-20T16:53:44,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8790 sec 2024-11-20T16:53:44,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.8850 sec 2024-11-20T16:53:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T16:53:45,137 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:45,137 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-20T16:53:45,141 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T16:53:45,142 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:45,142 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:45,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T16:53:45,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-20T16:53:45,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:45,294 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T16:53:45,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:53:45,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:45,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:53:45,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:45,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:53:45,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:45,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f89b2f027d214772a6633f2df6b91422_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121623649/Put/seqid=0 2024-11-20T16:53:45,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742217_1393 (size=12154) 2024-11-20T16:53:45,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:45,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:45,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:45,328 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f89b2f027d214772a6633f2df6b91422_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f89b2f027d214772a6633f2df6b91422_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:45,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/9d268cc08bc44f3a950a24aaadd09c00, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:45,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/9d268cc08bc44f3a950a24aaadd09c00 is 175, key is test_row_0/A:col10/1732121623649/Put/seqid=0 2024-11-20T16:53:45,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742218_1394 (size=30955) 2024-11-20T16:53:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T16:53:45,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121685462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:45,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:45,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:45,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:45,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121685576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:45,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:45,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:45,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:45,746 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/9d268cc08bc44f3a950a24aaadd09c00 2024-11-20T16:53:45,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/526ebd707c204a29bb44cb078075c1b8 is 50, key is test_row_0/B:col10/1732121623649/Put/seqid=0 2024-11-20T16:53:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T16:53:45,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121685797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:45,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742219_1395 (size=12001) 2024-11-20T16:53:45,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121685816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:45,819 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 5cb6af7207e610b3daed9e5466ca8411 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:45,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121685826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:45,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121685830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:45,838 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 5cb6af7207e610b3daed9e5466ca8411 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:45,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:45,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:45,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:45,843 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 5cb6af7207e610b3daed9e5466ca8411 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:45,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:45,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:45,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:45,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:45,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121685843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:45,854 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 5cb6af7207e610b3daed9e5466ca8411 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:45,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:45,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:45,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:46,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121686116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:46,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:46,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:46,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:46,208 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/526ebd707c204a29bb44cb078075c1b8 2024-11-20T16:53:46,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/724f293805f84cf5986235f7c8a33b61 is 50, key is test_row_0/C:col10/1732121623649/Put/seqid=0 2024-11-20T16:53:46,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742220_1396 (size=12001) 2024-11-20T16:53:46,224 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/724f293805f84cf5986235f7c8a33b61 2024-11-20T16:53:46,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/9d268cc08bc44f3a950a24aaadd09c00 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/9d268cc08bc44f3a950a24aaadd09c00 2024-11-20T16:53:46,233 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/9d268cc08bc44f3a950a24aaadd09c00, entries=150, sequenceid=97, filesize=30.2 K 2024-11-20T16:53:46,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/526ebd707c204a29bb44cb078075c1b8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/526ebd707c204a29bb44cb078075c1b8 2024-11-20T16:53:46,239 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/526ebd707c204a29bb44cb078075c1b8, entries=150, sequenceid=97, filesize=11.7 K 2024-11-20T16:53:46,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/724f293805f84cf5986235f7c8a33b61 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/724f293805f84cf5986235f7c8a33b61 2024-11-20T16:53:46,249 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/724f293805f84cf5986235f7c8a33b61, entries=150, sequenceid=97, filesize=11.7 K 2024-11-20T16:53:46,250 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5cb6af7207e610b3daed9e5466ca8411 in 956ms, sequenceid=97, compaction requested=true 2024-11-20T16:53:46,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:46,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:46,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-20T16:53:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-20T16:53:46,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-20T16:53:46,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1100 sec 2024-11-20T16:53:46,256 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.1180 sec 2024-11-20T16:53:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T16:53:46,277 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:46,277 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-20T16:53:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T16:53:46,280 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:46,282 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:46,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T16:53:46,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-20T16:53:46,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:46,435 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T16:53:46,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:53:46,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:46,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:53:46,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:46,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:53:46,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:46,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207fa18d2a0ced4b899c95f86f5e6c6721_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121625461/Put/seqid=0 2024-11-20T16:53:46,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742221_1397 (size=12154) 2024-11-20T16:53:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T16:53:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:46,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:46,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121686710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:46,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:46,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121686826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:46,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:46,855 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207fa18d2a0ced4b899c95f86f5e6c6721_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207fa18d2a0ced4b899c95f86f5e6c6721_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:46,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/aab16f6655104bf5a8f9fb42e2344857, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:46,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/aab16f6655104bf5a8f9fb42e2344857 is 175, key is test_row_0/A:col10/1732121625461/Put/seqid=0 2024-11-20T16:53:46,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742222_1398 (size=30955) 2024-11-20T16:53:46,868 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/aab16f6655104bf5a8f9fb42e2344857 2024-11-20T16:53:46,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/966aec3242bb4ccda4dc667eb9f3467e is 50, key is test_row_0/B:col10/1732121625461/Put/seqid=0 2024-11-20T16:53:46,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742223_1399 (size=12001) 2024-11-20T16:53:46,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T16:53:47,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:47,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121687035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:47,289 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/966aec3242bb4ccda4dc667eb9f3467e 2024-11-20T16:53:47,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/2f9575d1e7a14d5e87c3a7fc75a55aae is 50, key is test_row_0/C:col10/1732121625461/Put/seqid=0 2024-11-20T16:53:47,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742224_1400 (size=12001) 2024-11-20T16:53:47,308 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/2f9575d1e7a14d5e87c3a7fc75a55aae 2024-11-20T16:53:47,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/aab16f6655104bf5a8f9fb42e2344857 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/aab16f6655104bf5a8f9fb42e2344857 2024-11-20T16:53:47,322 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/aab16f6655104bf5a8f9fb42e2344857, entries=150, sequenceid=117, filesize=30.2 K 2024-11-20T16:53:47,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/966aec3242bb4ccda4dc667eb9f3467e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/966aec3242bb4ccda4dc667eb9f3467e 2024-11-20T16:53:47,330 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/966aec3242bb4ccda4dc667eb9f3467e, entries=150, sequenceid=117, filesize=11.7 K 2024-11-20T16:53:47,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/2f9575d1e7a14d5e87c3a7fc75a55aae as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/2f9575d1e7a14d5e87c3a7fc75a55aae 2024-11-20T16:53:47,348 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/2f9575d1e7a14d5e87c3a7fc75a55aae, entries=150, sequenceid=117, filesize=11.7 K 2024-11-20T16:53:47,349 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5cb6af7207e610b3daed9e5466ca8411 in 914ms, sequenceid=117, compaction requested=true 2024-11-20T16:53:47,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:47,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:47,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-20T16:53:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-20T16:53:47,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-20T16:53:47,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0690 sec 2024-11-20T16:53:47,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.0760 sec 2024-11-20T16:53:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:47,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T16:53:47,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:53:47,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:47,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:53:47,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:47,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:53:47,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:47,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f89d289943df473fabc95bd0482d6c3e_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121626709/Put/seqid=0 2024-11-20T16:53:47,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742225_1401 (size=14644) 2024-11-20T16:53:47,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T16:53:47,423 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:47,423 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-20T16:53:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T16:53:47,425 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:47,425 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:47,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T16:53:47,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:47,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:47,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:47,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:47,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:47,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:47,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121687579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:47,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:47,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:47,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:47,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121687696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:47,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:47,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:47,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:47,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:47,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:47,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T16:53:47,813 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:47,817 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f89d289943df473fabc95bd0482d6c3e_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f89d289943df473fabc95bd0482d6c3e_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:47,817 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/0d6b5ea88b58410f93bdcc8c4eed501c, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:47,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/0d6b5ea88b58410f93bdcc8c4eed501c is 175, key is test_row_0/A:col10/1732121626709/Put/seqid=0 2024-11-20T16:53:47,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121687836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:47,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742226_1402 (size=39599) 2024-11-20T16:53:47,843 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 5cb6af7207e610b3daed9e5466ca8411 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:47,845 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/0d6b5ea88b58410f93bdcc8c4eed501c 2024-11-20T16:53:47,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/c8c9b6e46380479aac256f05f0a6197a is 50, key is test_row_0/B:col10/1732121626709/Put/seqid=0 2024-11-20T16:53:47,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121687846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:47,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121687859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:47,868 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 5cb6af7207e610b3daed9e5466ca8411 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:47,868 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 5cb6af7207e610b3daed9e5466ca8411 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:47,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742227_1403 (size=12051) 2024-11-20T16:53:47,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/c8c9b6e46380479aac256f05f0a6197a 2024-11-20T16:53:47,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121687876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:47,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:47,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:47,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:47,889 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 5cb6af7207e610b3daed9e5466ca8411 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:47,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:47,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:47,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:47,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:47,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/974df1258f14436b8a6ea5240de0e274 is 50, key is test_row_0/C:col10/1732121626709/Put/seqid=0 2024-11-20T16:53:47,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742228_1404 (size=12051) 2024-11-20T16:53:47,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/974df1258f14436b8a6ea5240de0e274 2024-11-20T16:53:47,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/0d6b5ea88b58410f93bdcc8c4eed501c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0d6b5ea88b58410f93bdcc8c4eed501c 2024-11-20T16:53:47,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0d6b5ea88b58410f93bdcc8c4eed501c, entries=200, sequenceid=130, filesize=38.7 K 2024-11-20T16:53:47,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/c8c9b6e46380479aac256f05f0a6197a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/c8c9b6e46380479aac256f05f0a6197a 2024-11-20T16:53:47,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/c8c9b6e46380479aac256f05f0a6197a, entries=150, sequenceid=130, filesize=11.8 K 2024-11-20T16:53:47,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/974df1258f14436b8a6ea5240de0e274 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/974df1258f14436b8a6ea5240de0e274 2024-11-20T16:53:47,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/974df1258f14436b8a6ea5240de0e274, entries=150, sequenceid=130, filesize=11.8 K 2024-11-20T16:53:47,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5cb6af7207e610b3daed9e5466ca8411 in 557ms, sequenceid=130, compaction requested=true 2024-11-20T16:53:47,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:47,927 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T16:53:47,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:53:47,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:47,929 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 163522 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T16:53:47,929 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T16:53:47,929 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/A is initiating minor compaction (all files) 2024-11-20T16:53:47,929 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/A in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:47,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:53:47,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:47,929 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/e6e6154c27b44820b3b7677e326bc8e0, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/6139bcb0689d46bfa0dc1575d868d110, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/9d268cc08bc44f3a950a24aaadd09c00, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/aab16f6655104bf5a8f9fb42e2344857, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0d6b5ea88b58410f93bdcc8c4eed501c] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=159.7 K 2024-11-20T16:53:47,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:53:47,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:47,929 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:47,929 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/e6e6154c27b44820b3b7677e326bc8e0, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/6139bcb0689d46bfa0dc1575d868d110, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/9d268cc08bc44f3a950a24aaadd09c00, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/aab16f6655104bf5a8f9fb42e2344857, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0d6b5ea88b58410f93bdcc8c4eed501c] 2024-11-20T16:53:47,929 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6e6154c27b44820b3b7677e326bc8e0, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732121622251 2024-11-20T16:53:47,930 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6139bcb0689d46bfa0dc1575d868d110, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732121622930 2024-11-20T16:53:47,931 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d268cc08bc44f3a950a24aaadd09c00, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732121623634 2024-11-20T16:53:47,934 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60158 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T16:53:47,934 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/B is initiating minor compaction (all files) 2024-11-20T16:53:47,934 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/B in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:47,934 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/f969fc95faa240fcbdb9f291b087d00d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/a851663a9a4b4cc1850235b969292fe5, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/526ebd707c204a29bb44cb078075c1b8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/966aec3242bb4ccda4dc667eb9f3467e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/c8c9b6e46380479aac256f05f0a6197a] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=58.7 K 2024-11-20T16:53:47,935 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting aab16f6655104bf5a8f9fb42e2344857, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732121625426 2024-11-20T16:53:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:47,935 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting f969fc95faa240fcbdb9f291b087d00d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732121622251 2024-11-20T16:53:47,935 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d6b5ea88b58410f93bdcc8c4eed501c, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732121626684 2024-11-20T16:53:47,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T16:53:47,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:53:47,937 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting a851663a9a4b4cc1850235b969292fe5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732121622930 2024-11-20T16:53:47,937 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 526ebd707c204a29bb44cb078075c1b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732121623634 2024-11-20T16:53:47,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:47,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:53:47,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:47,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:53:47,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:47,937 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 966aec3242bb4ccda4dc667eb9f3467e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732121625426 2024-11-20T16:53:47,938 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting c8c9b6e46380479aac256f05f0a6197a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732121626684 2024-11-20T16:53:47,975 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:47,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112065f1a37be9454ca199ee63b7d1c57c87_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121627567/Put/seqid=0 2024-11-20T16:53:47,986 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#B#compaction#341 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:47,986 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/c902ec8d26cd4d22b94c0fa3b070ccd0 is 50, key is test_row_0/B:col10/1732121626709/Put/seqid=0 2024-11-20T16:53:47,991 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411201d451dd1dae146bf83a780ccf0cd7d10_5cb6af7207e610b3daed9e5466ca8411 store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:47,993 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411201d451dd1dae146bf83a780ccf0cd7d10_5cb6af7207e610b3daed9e5466ca8411, store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:47,993 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201d451dd1dae146bf83a780ccf0cd7d10_5cb6af7207e610b3daed9e5466ca8411 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:48,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742230_1406 (size=12325) 2024-11-20T16:53:48,025 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/c902ec8d26cd4d22b94c0fa3b070ccd0 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/c902ec8d26cd4d22b94c0fa3b070ccd0 2024-11-20T16:53:48,029 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/B of 5cb6af7207e610b3daed9e5466ca8411 into c902ec8d26cd4d22b94c0fa3b070ccd0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:48,029 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:48,029 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/B, priority=11, startTime=1732121627929; duration=0sec 2024-11-20T16:53:48,029 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:48,029 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:B 2024-11-20T16:53:48,029 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T16:53:48,031 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60158 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T16:53:48,031 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/C is initiating minor compaction (all files) 2024-11-20T16:53:48,031 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/C in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:48,032 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/e95ab6e3f36e483a8b680d4238c4e070, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/fc3993220a8c464ca4a1f7cea64c8aff, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/724f293805f84cf5986235f7c8a33b61, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/2f9575d1e7a14d5e87c3a7fc75a55aae, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/974df1258f14436b8a6ea5240de0e274] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=58.7 K 2024-11-20T16:53:48,032 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting e95ab6e3f36e483a8b680d4238c4e070, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732121622251 2024-11-20T16:53:48,032 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting fc3993220a8c464ca4a1f7cea64c8aff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732121622930 2024-11-20T16:53:48,033 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 724f293805f84cf5986235f7c8a33b61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732121623634 2024-11-20T16:53:48,033 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f9575d1e7a14d5e87c3a7fc75a55aae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732121625426 2024-11-20T16:53:48,033 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 974df1258f14436b8a6ea5240de0e274, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732121626684 2024-11-20T16:53:48,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:48,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:48,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:48,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:48,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742229_1405 (size=14794) 2024-11-20T16:53:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,047 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:48,050 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112065f1a37be9454ca199ee63b7d1c57c87_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112065f1a37be9454ca199ee63b7d1c57c87_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:48,051 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/ebedef411fa34465b3d9aabc99a240d9, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:48,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/ebedef411fa34465b3d9aabc99a240d9 is 175, key is test_row_0/A:col10/1732121627567/Put/seqid=0 2024-11-20T16:53:48,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742231_1407 (size=4469) 2024-11-20T16:53:48,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T16:53:48,057 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#A#compaction#340 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:48,057 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#C#compaction#343 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:48,057 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/0f6a42e733864a4ea53cbb706be237be is 175, key is test_row_0/A:col10/1732121626709/Put/seqid=0 2024-11-20T16:53:48,058 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/2752c6f2420a4905af7c5eb47354a2d2 is 50, key is test_row_0/C:col10/1732121626709/Put/seqid=0 2024-11-20T16:53:48,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121688047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:48,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:48,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:48,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:48,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742232_1408 (size=39749) 2024-11-20T16:53:48,069 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=150, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/ebedef411fa34465b3d9aabc99a240d9 2024-11-20T16:53:48,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/44f0d1f018614b99a2ee5815fb2a9ce5 is 50, key is test_row_0/B:col10/1732121627567/Put/seqid=0 2024-11-20T16:53:48,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742234_1410 (size=12325) 2024-11-20T16:53:48,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742233_1409 (size=31279) 2024-11-20T16:53:48,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742235_1411 (size=12151) 2024-11-20T16:53:48,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/44f0d1f018614b99a2ee5815fb2a9ce5 2024-11-20T16:53:48,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/047deefeffda4bd0a57841577021c16a is 50, key is test_row_0/C:col10/1732121627567/Put/seqid=0 2024-11-20T16:53:48,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742236_1412 (size=12151) 2024-11-20T16:53:48,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/047deefeffda4bd0a57841577021c16a 2024-11-20T16:53:48,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/ebedef411fa34465b3d9aabc99a240d9 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/ebedef411fa34465b3d9aabc99a240d9 2024-11-20T16:53:48,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/ebedef411fa34465b3d9aabc99a240d9, entries=200, sequenceid=150, filesize=38.8 K 2024-11-20T16:53:48,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/44f0d1f018614b99a2ee5815fb2a9ce5 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/44f0d1f018614b99a2ee5815fb2a9ce5 2024-11-20T16:53:48,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/44f0d1f018614b99a2ee5815fb2a9ce5, entries=150, sequenceid=150, filesize=11.9 K 2024-11-20T16:53:48,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/047deefeffda4bd0a57841577021c16a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/047deefeffda4bd0a57841577021c16a 2024-11-20T16:53:48,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/047deefeffda4bd0a57841577021c16a, entries=150, sequenceid=150, filesize=11.9 K 2024-11-20T16:53:48,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5cb6af7207e610b3daed9e5466ca8411 in 223ms, sequenceid=150, compaction requested=false 2024-11-20T16:53:48,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:48,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T16:53:48,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:53:48,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:48,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:53:48,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:48,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:53:48,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:48,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112031c221416f784d08bce9d3754807592f_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121628028/Put/seqid=0 2024-11-20T16:53:48,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:48,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:48,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:48,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:48,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742237_1413 (size=14794) 2024-11-20T16:53:48,230 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:48,234 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112031c221416f784d08bce9d3754807592f_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112031c221416f784d08bce9d3754807592f_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:48,234 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/fe9c767ae28e4635bcad8ad7df4be6a2, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:48,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/fe9c767ae28e4635bcad8ad7df4be6a2 is 175, key is test_row_0/A:col10/1732121628028/Put/seqid=0 2024-11-20T16:53:48,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742238_1414 (size=39749) 2024-11-20T16:53:48,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:48,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:48,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:48,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:48,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121688379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:48,501 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/2752c6f2420a4905af7c5eb47354a2d2 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/2752c6f2420a4905af7c5eb47354a2d2 2024-11-20T16:53:48,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:48,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:48,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:48,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:48,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,512 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/0f6a42e733864a4ea53cbb706be237be as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0f6a42e733864a4ea53cbb706be237be 2024-11-20T16:53:48,514 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/C of 5cb6af7207e610b3daed9e5466ca8411 into 2752c6f2420a4905af7c5eb47354a2d2(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:48,514 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:48,514 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/C, priority=11, startTime=1732121627929; duration=0sec 2024-11-20T16:53:48,514 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:48,514 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:C 2024-11-20T16:53:48,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121688508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:48,524 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/A of 5cb6af7207e610b3daed9e5466ca8411 into 0f6a42e733864a4ea53cbb706be237be(size=30.5 K), total size for store is 69.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:48,524 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:48,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:48,524 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/A, priority=11, startTime=1732121627927; duration=0sec 2024-11-20T16:53:48,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:48,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:48,524 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:48,524 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:A 2024-11-20T16:53:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T16:53:48,643 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=164, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/fe9c767ae28e4635bcad8ad7df4be6a2 2024-11-20T16:53:48,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/b1ad37d21a534ecaa60978128ba6cd46 is 50, key is test_row_0/B:col10/1732121628028/Put/seqid=0 2024-11-20T16:53:48,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:48,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:48,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:48,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:48,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742239_1415 (size=12151) 2024-11-20T16:53:48,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:48,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121688726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:48,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:48,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:48,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:48,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:48,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:48,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:48,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:48,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:48,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121689036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:49,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/b1ad37d21a534ecaa60978128ba6cd46 2024-11-20T16:53:49,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/4a10f493777843ef92902b6946b037b8 is 50, key is test_row_0/C:col10/1732121628028/Put/seqid=0 2024-11-20T16:53:49,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742240_1416 (size=12151) 2024-11-20T16:53:49,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:49,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:49,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:49,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:49,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:49,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:49,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:49,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:49,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:49,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:49,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:49,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:49,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/4a10f493777843ef92902b6946b037b8 2024-11-20T16:53:49,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/fe9c767ae28e4635bcad8ad7df4be6a2 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/fe9c767ae28e4635bcad8ad7df4be6a2 2024-11-20T16:53:49,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/fe9c767ae28e4635bcad8ad7df4be6a2, entries=200, sequenceid=164, filesize=38.8 K 2024-11-20T16:53:49,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/b1ad37d21a534ecaa60978128ba6cd46 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/b1ad37d21a534ecaa60978128ba6cd46 2024-11-20T16:53:49,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/b1ad37d21a534ecaa60978128ba6cd46, entries=150, sequenceid=164, filesize=11.9 K 2024-11-20T16:53:49,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/4a10f493777843ef92902b6946b037b8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/4a10f493777843ef92902b6946b037b8 2024-11-20T16:53:49,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/4a10f493777843ef92902b6946b037b8, entries=150, sequenceid=164, filesize=11.9 K 2024-11-20T16:53:49,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5cb6af7207e610b3daed9e5466ca8411 in 1340ms, sequenceid=164, compaction requested=true 2024-11-20T16:53:49,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:49,523 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:49,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:53:49,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:49,523 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:49,524 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110777 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:49,524 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/A is initiating minor compaction (all files) 2024-11-20T16:53:49,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:53:49,524 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/A in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:49,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:49,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:53:49,524 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0f6a42e733864a4ea53cbb706be237be, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/ebedef411fa34465b3d9aabc99a240d9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/fe9c767ae28e4635bcad8ad7df4be6a2] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=108.2 K 2024-11-20T16:53:49,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:49,524 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:49,524 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:49,524 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0f6a42e733864a4ea53cbb706be237be, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/ebedef411fa34465b3d9aabc99a240d9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/fe9c767ae28e4635bcad8ad7df4be6a2] 2024-11-20T16:53:49,524 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/B is initiating minor compaction (all files) 2024-11-20T16:53:49,524 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/B in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:49,524 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/c902ec8d26cd4d22b94c0fa3b070ccd0, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/44f0d1f018614b99a2ee5815fb2a9ce5, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/b1ad37d21a534ecaa60978128ba6cd46] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=35.8 K 2024-11-20T16:53:49,524 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f6a42e733864a4ea53cbb706be237be, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732121626684 2024-11-20T16:53:49,525 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting c902ec8d26cd4d22b94c0fa3b070ccd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732121626684 2024-11-20T16:53:49,525 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebedef411fa34465b3d9aabc99a240d9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732121627505 2024-11-20T16:53:49,525 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 44f0d1f018614b99a2ee5815fb2a9ce5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732121627567 2024-11-20T16:53:49,526 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe9c767ae28e4635bcad8ad7df4be6a2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732121627985 2024-11-20T16:53:49,526 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting b1ad37d21a534ecaa60978128ba6cd46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732121627985 2024-11-20T16:53:49,530 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:49,532 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411202b5d3c86eb4a46ba953f99c1a2fd8c38_5cb6af7207e610b3daed9e5466ca8411 store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:49,532 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#B#compaction#349 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:49,533 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/1b6256993eda4339b90a800a883bef83 is 50, key is test_row_0/B:col10/1732121628028/Put/seqid=0 2024-11-20T16:53:49,534 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411202b5d3c86eb4a46ba953f99c1a2fd8c38_5cb6af7207e610b3daed9e5466ca8411, store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:49,534 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202b5d3c86eb4a46ba953f99c1a2fd8c38_5cb6af7207e610b3daed9e5466ca8411 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:49,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742241_1417 (size=12527) 2024-11-20T16:53:49,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742242_1418 (size=4469) 2024-11-20T16:53:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:49,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T16:53:49,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:53:49,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:49,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:53:49,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:49,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:53:49,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:49,554 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#A#compaction#350 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:49,555 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/c88946248c5241029c108fc4e881e43a is 175, key is test_row_0/A:col10/1732121628028/Put/seqid=0 2024-11-20T16:53:49,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742243_1419 (size=31481) 2024-11-20T16:53:49,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207f96696f70de432e875f9ed602a8f208_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121629551/Put/seqid=0 2024-11-20T16:53:49,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742244_1420 (size=14794) 2024-11-20T16:53:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T16:53:49,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:49,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:49,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:49,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:49,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121689618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:49,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121689736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:49,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:49,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:49,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:49,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:49,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:49,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:49,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:49,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:49,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:49,947 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/1b6256993eda4339b90a800a883bef83 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/1b6256993eda4339b90a800a883bef83 2024-11-20T16:53:49,955 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/B of 5cb6af7207e610b3daed9e5466ca8411 into 1b6256993eda4339b90a800a883bef83(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:49,955 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:49,955 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/B, priority=13, startTime=1732121629523; duration=0sec 2024-11-20T16:53:49,955 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:49,955 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:B 2024-11-20T16:53:49,955 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:49,956 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:49,956 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/C is initiating minor compaction (all files) 2024-11-20T16:53:49,956 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/C in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:49,956 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/2752c6f2420a4905af7c5eb47354a2d2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/047deefeffda4bd0a57841577021c16a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/4a10f493777843ef92902b6946b037b8] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=35.8 K 2024-11-20T16:53:49,957 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 2752c6f2420a4905af7c5eb47354a2d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732121626684 2024-11-20T16:53:49,957 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 047deefeffda4bd0a57841577021c16a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732121627567 2024-11-20T16:53:49,958 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a10f493777843ef92902b6946b037b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732121627985 2024-11-20T16:53:49,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121689946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:49,965 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/c88946248c5241029c108fc4e881e43a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c88946248c5241029c108fc4e881e43a 2024-11-20T16:53:49,970 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/A of 5cb6af7207e610b3daed9e5466ca8411 into c88946248c5241029c108fc4e881e43a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:49,970 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:49,970 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/A, priority=13, startTime=1732121629523; duration=0sec 2024-11-20T16:53:49,970 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:49,970 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:A 2024-11-20T16:53:49,975 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:49,981 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#C#compaction#352 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:49,981 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/cbbcfed0db0147b0968d9910089ab0f6 is 50, key is test_row_0/C:col10/1732121628028/Put/seqid=0 2024-11-20T16:53:49,985 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207f96696f70de432e875f9ed602a8f208_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207f96696f70de432e875f9ed602a8f208_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:49,986 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/c5adb3b84f4541dda19e287a5478cf90, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:49,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/c5adb3b84f4541dda19e287a5478cf90 is 175, key is test_row_0/A:col10/1732121629551/Put/seqid=0 2024-11-20T16:53:50,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742245_1421 (size=12527) 2024-11-20T16:53:50,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742246_1422 (size=39749) 2024-11-20T16:53:50,011 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/c5adb3b84f4541dda19e287a5478cf90 2024-11-20T16:53:50,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/e7e543333c7b4b9cb5a2ec1755dd4157 is 50, key is test_row_0/B:col10/1732121629551/Put/seqid=0 2024-11-20T16:53:50,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742247_1423 (size=12151) 2024-11-20T16:53:50,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/e7e543333c7b4b9cb5a2ec1755dd4157 2024-11-20T16:53:50,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:50,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:50,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:50,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:50,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:50,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:50,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/1e691ca32de84a1b9d208577adde8498 is 50, key is test_row_0/C:col10/1732121629551/Put/seqid=0 2024-11-20T16:53:50,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742248_1424 (size=12151) 2024-11-20T16:53:50,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:50,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:50,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:50,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:50,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121690266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:50,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:50,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:50,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:50,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:50,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:50,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:50,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:50,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:50,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:50,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:50,409 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/cbbcfed0db0147b0968d9910089ab0f6 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/cbbcfed0db0147b0968d9910089ab0f6 2024-11-20T16:53:50,417 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/C of 5cb6af7207e610b3daed9e5466ca8411 into cbbcfed0db0147b0968d9910089ab0f6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:50,417 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:50,417 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/C, priority=13, startTime=1732121629524; duration=0sec 2024-11-20T16:53:50,417 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:50,417 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:C 2024-11-20T16:53:50,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/1e691ca32de84a1b9d208577adde8498 2024-11-20T16:53:50,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/c5adb3b84f4541dda19e287a5478cf90 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c5adb3b84f4541dda19e287a5478cf90 2024-11-20T16:53:50,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c5adb3b84f4541dda19e287a5478cf90, entries=200, sequenceid=186, filesize=38.8 K 2024-11-20T16:53:50,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/e7e543333c7b4b9cb5a2ec1755dd4157 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/e7e543333c7b4b9cb5a2ec1755dd4157 2024-11-20T16:53:50,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/e7e543333c7b4b9cb5a2ec1755dd4157, entries=150, sequenceid=186, filesize=11.9 K 2024-11-20T16:53:50,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/1e691ca32de84a1b9d208577adde8498 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/1e691ca32de84a1b9d208577adde8498 2024-11-20T16:53:50,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/1e691ca32de84a1b9d208577adde8498, entries=150, sequenceid=186, filesize=11.9 K 2024-11-20T16:53:50,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5cb6af7207e610b3daed9e5466ca8411 in 936ms, sequenceid=186, compaction requested=false 2024-11-20T16:53:50,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:50,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T16:53:50,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:50,512 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T16:53:50,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:53:50,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:50,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:53:50,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:50,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:53:50,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:50,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112017f75dfd53074ab7bf58d691d0927746_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121629615/Put/seqid=0 2024-11-20T16:53:50,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742249_1425 (size=12304) 2024-11-20T16:53:50,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:50,554 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112017f75dfd53074ab7bf58d691d0927746_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112017f75dfd53074ab7bf58d691d0927746_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:50,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/d9648ba0514442b9841f2d92db3ce7cd, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:50,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/d9648ba0514442b9841f2d92db3ce7cd is 175, key is test_row_0/A:col10/1732121629615/Put/seqid=0 2024-11-20T16:53:50,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742250_1426 (size=31105) 2024-11-20T16:53:50,560 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/d9648ba0514442b9841f2d92db3ce7cd 2024-11-20T16:53:50,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/6ad2ec633c7946fc8aaffa15bc01ccf3 is 50, key is test_row_0/B:col10/1732121629615/Put/seqid=0 2024-11-20T16:53:50,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742251_1427 (size=12151) 2024-11-20T16:53:50,594 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/6ad2ec633c7946fc8aaffa15bc01ccf3 2024-11-20T16:53:50,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/6cd2f4fb7ab34d249ea1975c5d69a98d is 50, key is test_row_0/C:col10/1732121629615/Put/seqid=0 2024-11-20T16:53:50,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742252_1428 (size=12151) 2024-11-20T16:53:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:50,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:50,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121690879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:50,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:50,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:50,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:51,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121690995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:51,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:51,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:51,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:51,032 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/6cd2f4fb7ab34d249ea1975c5d69a98d 2024-11-20T16:53:51,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/d9648ba0514442b9841f2d92db3ce7cd as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/d9648ba0514442b9841f2d92db3ce7cd 2024-11-20T16:53:51,040 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/d9648ba0514442b9841f2d92db3ce7cd, entries=150, sequenceid=201, filesize=30.4 K 2024-11-20T16:53:51,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/6ad2ec633c7946fc8aaffa15bc01ccf3 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/6ad2ec633c7946fc8aaffa15bc01ccf3 2024-11-20T16:53:51,044 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/6ad2ec633c7946fc8aaffa15bc01ccf3, entries=150, sequenceid=201, filesize=11.9 K 2024-11-20T16:53:51,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/6cd2f4fb7ab34d249ea1975c5d69a98d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/6cd2f4fb7ab34d249ea1975c5d69a98d 2024-11-20T16:53:51,048 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/6cd2f4fb7ab34d249ea1975c5d69a98d, entries=150, sequenceid=201, filesize=11.9 K 2024-11-20T16:53:51,049 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5cb6af7207e610b3daed9e5466ca8411 in 537ms, sequenceid=201, compaction requested=true 2024-11-20T16:53:51,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:51,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:51,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-20T16:53:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-20T16:53:51,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-20T16:53:51,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6250 sec 2024-11-20T16:53:51,053 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 3.6290 sec 2024-11-20T16:53:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:51,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-20T16:53:51,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:53:51,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:51,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:53:51,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:51,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:53:51,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:51,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206248512482394aedbdd4577b55eeb312_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121630877/Put/seqid=0 2024-11-20T16:53:51,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742253_1429 (size=14794) 2024-11-20T16:53:51,250 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:51,254 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206248512482394aedbdd4577b55eeb312_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206248512482394aedbdd4577b55eeb312_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:51,255 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/ec764d8478dc40c3908e2a74b253fc24, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:51,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/ec764d8478dc40c3908e2a74b253fc24 is 175, key is test_row_0/A:col10/1732121630877/Put/seqid=0 2024-11-20T16:53:51,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742254_1430 (size=39749) 2024-11-20T16:53:51,274 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/ec764d8478dc40c3908e2a74b253fc24 2024-11-20T16:53:51,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/4502c1fd47894a608722355df5ea1bd8 is 50, key is test_row_0/B:col10/1732121630877/Put/seqid=0 2024-11-20T16:53:51,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742255_1431 (size=12151) 2024-11-20T16:53:51,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121691308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:51,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121691426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T16:53:51,587 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:51,587 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-20T16:53:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T16:53:51,589 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:51,589 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:51,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:51,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121691646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:51,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:51,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:51,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:51,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/4502c1fd47894a608722355df5ea1bd8 2024-11-20T16:53:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T16:53:51,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/138aa0407c704eeaa4f81b93df37f673 is 50, key is test_row_0/C:col10/1732121630877/Put/seqid=0 2024-11-20T16:53:51,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742256_1432 (size=12151) 2024-11-20T16:53:51,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T16:53:51,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:51,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:51,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:51,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:51,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:51,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121691867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:51,880 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 5cb6af7207e610b3daed9e5466ca8411 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:51,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:51,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121691876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:51,890 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 5cb6af7207e610b3daed9e5466ca8411 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:51,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:51,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:51,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:51,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121691887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:51,900 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 5cb6af7207e610b3daed9e5466ca8411 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T16:53:51,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T16:53:51,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:51,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:51,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:51,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:51,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:51,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121691927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:51,936 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 5cb6af7207e610b3daed9e5466ca8411 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8285 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:53:51,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:51,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121691966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:51,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:51,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:51,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:52,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T16:53:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:52,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:52,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/138aa0407c704eeaa4f81b93df37f673 2024-11-20T16:53:52,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/ec764d8478dc40c3908e2a74b253fc24 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/ec764d8478dc40c3908e2a74b253fc24 2024-11-20T16:53:52,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/ec764d8478dc40c3908e2a74b253fc24, entries=200, sequenceid=222, filesize=38.8 K 2024-11-20T16:53:52,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/4502c1fd47894a608722355df5ea1bd8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/4502c1fd47894a608722355df5ea1bd8 2024-11-20T16:53:52,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/4502c1fd47894a608722355df5ea1bd8, entries=150, sequenceid=222, filesize=11.9 K 2024-11-20T16:53:52,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/138aa0407c704eeaa4f81b93df37f673 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/138aa0407c704eeaa4f81b93df37f673 2024-11-20T16:53:52,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/138aa0407c704eeaa4f81b93df37f673, entries=150, sequenceid=222, filesize=11.9 K 2024-11-20T16:53:52,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 5cb6af7207e610b3daed9e5466ca8411 in 906ms, sequenceid=222, compaction requested=true 2024-11-20T16:53:52,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:52,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:53:52,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:52,135 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:53:52,135 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:53:52,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:53:52,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:52,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:53:52,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:52,136 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:53:52,136 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/B is initiating minor compaction (all files) 2024-11-20T16:53:52,136 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/B in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:52,136 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/1b6256993eda4339b90a800a883bef83, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/e7e543333c7b4b9cb5a2ec1755dd4157, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/6ad2ec633c7946fc8aaffa15bc01ccf3, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/4502c1fd47894a608722355df5ea1bd8] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=47.8 K 2024-11-20T16:53:52,136 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142084 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:53:52,136 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/A is initiating minor compaction (all files) 2024-11-20T16:53:52,136 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/A in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:52,137 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c88946248c5241029c108fc4e881e43a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c5adb3b84f4541dda19e287a5478cf90, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/d9648ba0514442b9841f2d92db3ce7cd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/ec764d8478dc40c3908e2a74b253fc24] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=138.8 K 2024-11-20T16:53:52,137 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b6256993eda4339b90a800a883bef83, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732121627985 2024-11-20T16:53:52,137 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:52,137 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c88946248c5241029c108fc4e881e43a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c5adb3b84f4541dda19e287a5478cf90, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/d9648ba0514442b9841f2d92db3ce7cd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/ec764d8478dc40c3908e2a74b253fc24] 2024-11-20T16:53:52,137 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting e7e543333c7b4b9cb5a2ec1755dd4157, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732121628297 2024-11-20T16:53:52,137 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting c88946248c5241029c108fc4e881e43a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732121627985 2024-11-20T16:53:52,137 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ad2ec633c7946fc8aaffa15bc01ccf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732121629591 2024-11-20T16:53:52,137 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5adb3b84f4541dda19e287a5478cf90, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732121628297 2024-11-20T16:53:52,137 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 4502c1fd47894a608722355df5ea1bd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732121630864 2024-11-20T16:53:52,137 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9648ba0514442b9841f2d92db3ce7cd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732121629591 2024-11-20T16:53:52,138 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec764d8478dc40c3908e2a74b253fc24, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732121630812 2024-11-20T16:53:52,150 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#B#compaction#361 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:52,150 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/d327fe367f6d4263aa7d71d9219385bb is 50, key is test_row_0/B:col10/1732121630877/Put/seqid=0 2024-11-20T16:53:52,151 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:52,153 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411201700468ee6184a33af7df8e90269b81e_5cb6af7207e610b3daed9e5466ca8411 store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:52,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742257_1433 (size=12663) 2024-11-20T16:53:52,156 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411201700468ee6184a33af7df8e90269b81e_5cb6af7207e610b3daed9e5466ca8411, store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:52,156 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201700468ee6184a33af7df8e90269b81e_5cb6af7207e610b3daed9e5466ca8411 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:52,159 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/d327fe367f6d4263aa7d71d9219385bb as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/d327fe367f6d4263aa7d71d9219385bb 2024-11-20T16:53:52,162 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/B of 5cb6af7207e610b3daed9e5466ca8411 into d327fe367f6d4263aa7d71d9219385bb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:52,162 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:52,162 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/B, priority=12, startTime=1732121632135; duration=0sec 2024-11-20T16:53:52,162 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:52,162 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:B 2024-11-20T16:53:52,162 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:53:52,163 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:53:52,163 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/C is initiating minor compaction (all files) 2024-11-20T16:53:52,163 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/C in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:52,163 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/cbbcfed0db0147b0968d9910089ab0f6, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/1e691ca32de84a1b9d208577adde8498, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/6cd2f4fb7ab34d249ea1975c5d69a98d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/138aa0407c704eeaa4f81b93df37f673] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=47.8 K 2024-11-20T16:53:52,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742258_1434 (size=4469) 2024-11-20T16:53:52,166 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting cbbcfed0db0147b0968d9910089ab0f6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732121627985 2024-11-20T16:53:52,166 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e691ca32de84a1b9d208577adde8498, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732121628297 2024-11-20T16:53:52,166 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cd2f4fb7ab34d249ea1975c5d69a98d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732121629591 2024-11-20T16:53:52,167 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 138aa0407c704eeaa4f81b93df37f673, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732121630864 2024-11-20T16:53:52,175 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#C#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:52,175 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/f913ab27ef54489998a526f312770532 is 50, key is test_row_0/C:col10/1732121630877/Put/seqid=0 2024-11-20T16:53:52,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742259_1435 (size=12663) 2024-11-20T16:53:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T16:53:52,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T16:53:52,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:52,216 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T16:53:52,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:53:52,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:52,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:53:52,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:52,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:53:52,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:52,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205a7aea95d2434564996755175da72ef1_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121631302/Put/seqid=0 2024-11-20T16:53:52,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742260_1436 (size=12304) 2024-11-20T16:53:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:52,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:52,564 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#A#compaction#362 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:52,565 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/1ea4663de0b04135ad6c0842a259150e is 175, key is test_row_0/A:col10/1732121630877/Put/seqid=0 2024-11-20T16:53:52,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742261_1437 (size=31617) 2024-11-20T16:53:52,586 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/f913ab27ef54489998a526f312770532 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/f913ab27ef54489998a526f312770532 2024-11-20T16:53:52,589 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/C of 5cb6af7207e610b3daed9e5466ca8411 into f913ab27ef54489998a526f312770532(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:52,589 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:52,589 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/C, priority=12, startTime=1732121632135; duration=0sec 2024-11-20T16:53:52,589 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:52,589 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:C 2024-11-20T16:53:52,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:52,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121632591, max timestamp=1732121632627Num uniques 859; 2024-11-20T16:53:52,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121632591, max timestamp=1732121632627Num uniques 859; 2024-11-20T16:53:52,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:53:52,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121632591, max timestamp=1732121632627Num uniques 859; 2024-11-20T16:53:52,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:53:52,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:53:52,631 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205a7aea95d2434564996755175da72ef1_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205a7aea95d2434564996755175da72ef1_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/6047c26f6f2044bf9835e2f2da6fb11b, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/6047c26f6f2044bf9835e2f2da6fb11b is 175, key is test_row_0/A:col10/1732121631302/Put/seqid=0 2024-11-20T16:53:52,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742262_1438 (size=31105) 2024-11-20T16:53:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T16:53:52,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121692775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:52,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121692886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:52,972 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/1ea4663de0b04135ad6c0842a259150e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1ea4663de0b04135ad6c0842a259150e 2024-11-20T16:53:52,976 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/A of 5cb6af7207e610b3daed9e5466ca8411 into 1ea4663de0b04135ad6c0842a259150e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:52,976 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:52,976 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/A, priority=12, startTime=1732121632135; duration=0sec 2024-11-20T16:53:52,977 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:52,977 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:A 2024-11-20T16:53:53,036 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/6047c26f6f2044bf9835e2f2da6fb11b 2024-11-20T16:53:53,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/7bd61a8bc5d4463da93fe5089dc47154 is 50, key is test_row_0/B:col10/1732121631302/Put/seqid=0 2024-11-20T16:53:53,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742263_1439 (size=12151) 2024-11-20T16:53:53,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121693106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:53,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:53,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:53,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:53,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:53,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121693426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:53,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:53,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:53,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:53,448 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/7bd61a8bc5d4463da93fe5089dc47154 2024-11-20T16:53:53,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/b18252c2852847f69df201b92db8a22a is 50, key is test_row_0/C:col10/1732121631302/Put/seqid=0 2024-11-20T16:53:53,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742264_1440 (size=12151) 2024-11-20T16:53:53,457 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/b18252c2852847f69df201b92db8a22a 2024-11-20T16:53:53,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/6047c26f6f2044bf9835e2f2da6fb11b as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/6047c26f6f2044bf9835e2f2da6fb11b 2024-11-20T16:53:53,469 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/6047c26f6f2044bf9835e2f2da6fb11b, entries=150, sequenceid=234, filesize=30.4 K 2024-11-20T16:53:53,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/7bd61a8bc5d4463da93fe5089dc47154 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/7bd61a8bc5d4463da93fe5089dc47154 2024-11-20T16:53:53,473 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/7bd61a8bc5d4463da93fe5089dc47154, entries=150, sequenceid=234, filesize=11.9 K 2024-11-20T16:53:53,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/b18252c2852847f69df201b92db8a22a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/b18252c2852847f69df201b92db8a22a 2024-11-20T16:53:53,477 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/b18252c2852847f69df201b92db8a22a, entries=150, sequenceid=234, filesize=11.9 K 2024-11-20T16:53:53,478 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5cb6af7207e610b3daed9e5466ca8411 in 1261ms, sequenceid=234, compaction requested=false 2024-11-20T16:53:53,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:53,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:53,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-20T16:53:53,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-20T16:53:53,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-20T16:53:53,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8900 sec 2024-11-20T16:53:53,483 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.8950 sec 2024-11-20T16:53:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T16:53:53,736 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:53,736 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-20T16:53:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T16:53:53,737 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:53,738 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:53,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T16:53:53,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-20T16:53:53,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:53,891 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-20T16:53:53,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:53:53,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:53,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:53:53,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:53,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:53:53,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:53:53,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112077d70475dbfb49d7ab4bfba1f824a4d4_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121632760/Put/seqid=0 2024-11-20T16:53:53,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742265_1441 (size=12454) 2024-11-20T16:53:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:53,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:54,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121694013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T16:53:54,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121694126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:54,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:54,315 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112077d70475dbfb49d7ab4bfba1f824a4d4_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112077d70475dbfb49d7ab4bfba1f824a4d4_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:54,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/9ad804bd1fb8435d8d77a2979a18e04d, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:54,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/9ad804bd1fb8435d8d77a2979a18e04d is 175, key is test_row_0/A:col10/1732121632760/Put/seqid=0 2024-11-20T16:53:54,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742266_1442 (size=31255) 2024-11-20T16:53:54,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121694346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T16:53:54,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121694656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:54,720 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/9ad804bd1fb8435d8d77a2979a18e04d 2024-11-20T16:53:54,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/1f58de598a7e4fbf9c05be1ffa6756c4 is 50, key is test_row_0/B:col10/1732121632760/Put/seqid=0 2024-11-20T16:53:54,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742267_1443 (size=12301) 2024-11-20T16:53:54,751 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/1f58de598a7e4fbf9c05be1ffa6756c4 2024-11-20T16:53:54,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/01e2a5d420504680a994c38a4b190af9 is 50, key is test_row_0/C:col10/1732121632760/Put/seqid=0 2024-11-20T16:53:54,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742268_1444 (size=12301) 2024-11-20T16:53:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T16:53:55,164 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/01e2a5d420504680a994c38a4b190af9 2024-11-20T16:53:55,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/9ad804bd1fb8435d8d77a2979a18e04d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/9ad804bd1fb8435d8d77a2979a18e04d 2024-11-20T16:53:55,170 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/9ad804bd1fb8435d8d77a2979a18e04d, entries=150, sequenceid=272, filesize=30.5 K 2024-11-20T16:53:55,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/1f58de598a7e4fbf9c05be1ffa6756c4 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/1f58de598a7e4fbf9c05be1ffa6756c4 2024-11-20T16:53:55,173 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/1f58de598a7e4fbf9c05be1ffa6756c4, entries=150, sequenceid=272, filesize=12.0 K 2024-11-20T16:53:55,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/01e2a5d420504680a994c38a4b190af9 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/01e2a5d420504680a994c38a4b190af9 2024-11-20T16:53:55,177 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/01e2a5d420504680a994c38a4b190af9, entries=150, sequenceid=272, filesize=12.0 K 2024-11-20T16:53:55,177 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 5cb6af7207e610b3daed9e5466ca8411 in 1286ms, sequenceid=272, compaction requested=true 2024-11-20T16:53:55,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:55,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:55,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-20T16:53:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-20T16:53:55,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-20T16:53:55,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4400 sec 2024-11-20T16:53:55,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.4440 sec 2024-11-20T16:53:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:55,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T16:53:55,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:53:55,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:55,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:53:55,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:55,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:53:55,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:55,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e63ce0ced1174997a822d2dab3bc2b6c_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121633994/Put/seqid=0 2024-11-20T16:53:55,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742269_1445 (size=14994) 2024-11-20T16:53:55,199 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:55,201 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e63ce0ced1174997a822d2dab3bc2b6c_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e63ce0ced1174997a822d2dab3bc2b6c_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:55,202 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/b98f1409dc144293b661f07432ea2805, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:55,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/b98f1409dc144293b661f07432ea2805 is 175, key is test_row_0/A:col10/1732121633994/Put/seqid=0 2024-11-20T16:53:55,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742270_1446 (size=39949) 2024-11-20T16:53:55,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121695356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:55,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121695466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:55,606 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/b98f1409dc144293b661f07432ea2805 2024-11-20T16:53:55,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/5ee24590dcf34c33a42ae22605cd4d56 is 50, key is test_row_0/B:col10/1732121633994/Put/seqid=0 2024-11-20T16:53:55,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742271_1447 (size=12301) 2024-11-20T16:53:55,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/5ee24590dcf34c33a42ae22605cd4d56 2024-11-20T16:53:55,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/ce7ba37e1912404d81e87359892efb12 is 50, key is test_row_0/C:col10/1732121633994/Put/seqid=0 2024-11-20T16:53:55,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742272_1448 (size=12301) 2024-11-20T16:53:55,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121695686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T16:53:55,886 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:55,887 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-20T16:53:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T16:53:55,888 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:55,889 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:55,889 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T16:53:56,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121696006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:56,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T16:53:56,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:56,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:56,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:56,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:56,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:56,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/ce7ba37e1912404d81e87359892efb12 2024-11-20T16:53:56,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/b98f1409dc144293b661f07432ea2805 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/b98f1409dc144293b661f07432ea2805 2024-11-20T16:53:56,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/b98f1409dc144293b661f07432ea2805, entries=200, sequenceid=284, filesize=39.0 K 2024-11-20T16:53:56,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/5ee24590dcf34c33a42ae22605cd4d56 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/5ee24590dcf34c33a42ae22605cd4d56 2024-11-20T16:53:56,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/5ee24590dcf34c33a42ae22605cd4d56, entries=150, sequenceid=284, filesize=12.0 K 2024-11-20T16:53:56,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/ce7ba37e1912404d81e87359892efb12 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/ce7ba37e1912404d81e87359892efb12 2024-11-20T16:53:56,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/ce7ba37e1912404d81e87359892efb12, entries=150, sequenceid=284, filesize=12.0 K 2024-11-20T16:53:56,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5cb6af7207e610b3daed9e5466ca8411 in 883ms, sequenceid=284, compaction requested=true 2024-11-20T16:53:56,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:56,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:53:56,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:56,066 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:53:56,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:53:56,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:56,066 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:53:56,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:53:56,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:56,067 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:53:56,067 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133926 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:53:56,067 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/B is initiating minor compaction (all files) 2024-11-20T16:53:56,067 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/A is initiating minor compaction (all files) 2024-11-20T16:53:56,067 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/A in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:56,067 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/B in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:56,067 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/d327fe367f6d4263aa7d71d9219385bb, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/7bd61a8bc5d4463da93fe5089dc47154, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/1f58de598a7e4fbf9c05be1ffa6756c4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/5ee24590dcf34c33a42ae22605cd4d56] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=48.3 K 2024-11-20T16:53:56,067 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1ea4663de0b04135ad6c0842a259150e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/6047c26f6f2044bf9835e2f2da6fb11b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/9ad804bd1fb8435d8d77a2979a18e04d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/b98f1409dc144293b661f07432ea2805] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=130.8 K 2024-11-20T16:53:56,067 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:56,067 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1ea4663de0b04135ad6c0842a259150e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/6047c26f6f2044bf9835e2f2da6fb11b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/9ad804bd1fb8435d8d77a2979a18e04d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/b98f1409dc144293b661f07432ea2805] 2024-11-20T16:53:56,068 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting d327fe367f6d4263aa7d71d9219385bb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732121630864 2024-11-20T16:53:56,068 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ea4663de0b04135ad6c0842a259150e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732121630864 2024-11-20T16:53:56,068 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bd61a8bc5d4463da93fe5089dc47154, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732121631251 2024-11-20T16:53:56,068 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6047c26f6f2044bf9835e2f2da6fb11b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732121631251 2024-11-20T16:53:56,068 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f58de598a7e4fbf9c05be1ffa6756c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732121632690 2024-11-20T16:53:56,068 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ad804bd1fb8435d8d77a2979a18e04d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732121632690 2024-11-20T16:53:56,069 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ee24590dcf34c33a42ae22605cd4d56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732121633994 2024-11-20T16:53:56,069 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting b98f1409dc144293b661f07432ea2805, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732121633984 2024-11-20T16:53:56,079 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#B#compaction#373 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:56,080 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:56,080 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/b3f3731f53c7425099a96cf17e4d0f18 is 50, key is test_row_0/B:col10/1732121633994/Put/seqid=0 2024-11-20T16:53:56,083 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411207d25c72e9e304125b7d5bdf81f03ab6a_5cb6af7207e610b3daed9e5466ca8411 store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:56,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742273_1449 (size=12949) 2024-11-20T16:53:56,086 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411207d25c72e9e304125b7d5bdf81f03ab6a_5cb6af7207e610b3daed9e5466ca8411, store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:56,087 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207d25c72e9e304125b7d5bdf81f03ab6a_5cb6af7207e610b3daed9e5466ca8411 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:56,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742274_1450 (size=4469) 2024-11-20T16:53:56,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T16:53:56,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:56,194 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T16:53:56,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:53:56,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:56,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:53:56,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:56,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:53:56,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:56,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e2efce93a0b74839b74bf713ec2dbcac_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121635339/Put/seqid=0 2024-11-20T16:53:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T16:53:56,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742275_1451 (size=12454) 2024-11-20T16:53:56,503 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#A#compaction#374 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:56,503 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/1ec13824234e45798a49657a45ef1716 is 175, key is test_row_0/A:col10/1732121633994/Put/seqid=0 2024-11-20T16:53:56,514 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/b3f3731f53c7425099a96cf17e4d0f18 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/b3f3731f53c7425099a96cf17e4d0f18 2024-11-20T16:53:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T16:53:56,518 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/B of 5cb6af7207e610b3daed9e5466ca8411 into b3f3731f53c7425099a96cf17e4d0f18(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:56,518 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:56,518 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/B, priority=12, startTime=1732121636066; duration=0sec 2024-11-20T16:53:56,518 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:56,518 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:B 2024-11-20T16:53:56,518 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:53:56,519 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:53:56,519 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/C is initiating minor compaction (all files) 2024-11-20T16:53:56,519 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/C in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:56,519 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/f913ab27ef54489998a526f312770532, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/b18252c2852847f69df201b92db8a22a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/01e2a5d420504680a994c38a4b190af9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/ce7ba37e1912404d81e87359892efb12] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=48.3 K 2024-11-20T16:53:56,519 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting f913ab27ef54489998a526f312770532, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732121630864 2024-11-20T16:53:56,519 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting b18252c2852847f69df201b92db8a22a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732121631251 2024-11-20T16:53:56,520 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 01e2a5d420504680a994c38a4b190af9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732121632690 2024-11-20T16:53:56,520 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting ce7ba37e1912404d81e87359892efb12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732121633994 2024-11-20T16:53:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:56,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:56,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742276_1452 (size=31903) 2024-11-20T16:53:56,538 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/1ec13824234e45798a49657a45ef1716 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1ec13824234e45798a49657a45ef1716 2024-11-20T16:53:56,538 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#C#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:56,538 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/ae5a2364ded14e9b952cd79391d13011 is 50, key is test_row_0/C:col10/1732121633994/Put/seqid=0 2024-11-20T16:53:56,541 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/A of 5cb6af7207e610b3daed9e5466ca8411 into 1ec13824234e45798a49657a45ef1716(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:56,541 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:56,542 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/A, priority=12, startTime=1732121636066; duration=0sec 2024-11-20T16:53:56,542 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:56,542 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:A 2024-11-20T16:53:56,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742277_1453 (size=12949) 2024-11-20T16:53:56,550 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/ae5a2364ded14e9b952cd79391d13011 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/ae5a2364ded14e9b952cd79391d13011 2024-11-20T16:53:56,554 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/C of 5cb6af7207e610b3daed9e5466ca8411 into ae5a2364ded14e9b952cd79391d13011(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:56,554 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:56,555 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/C, priority=12, startTime=1732121636066; duration=0sec 2024-11-20T16:53:56,555 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:56,555 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:C 2024-11-20T16:53:56,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121696588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:56,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:56,617 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e2efce93a0b74839b74bf713ec2dbcac_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e2efce93a0b74839b74bf713ec2dbcac_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:56,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/befe950e4c074302aa885ed1213c5459, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:56,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/befe950e4c074302aa885ed1213c5459 is 175, key is test_row_0/A:col10/1732121635339/Put/seqid=0 2024-11-20T16:53:56,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742278_1454 (size=31255) 2024-11-20T16:53:56,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121696696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:56,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121696906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:56,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:56,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:56,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:57,023 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/befe950e4c074302aa885ed1213c5459 2024-11-20T16:53:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T16:53:57,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/e87c8e48e574452889c0baaa05a27b2d is 50, key is test_row_0/B:col10/1732121635339/Put/seqid=0 2024-11-20T16:53:57,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742279_1455 (size=12301) 2024-11-20T16:53:57,032 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/e87c8e48e574452889c0baaa05a27b2d 2024-11-20T16:53:57,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/157c2172a1fd48b9a44ec676c682ddde is 50, key is test_row_0/C:col10/1732121635339/Put/seqid=0 2024-11-20T16:53:57,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742280_1456 (size=12301) 2024-11-20T16:53:57,046 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/157c2172a1fd48b9a44ec676c682ddde 2024-11-20T16:53:57,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/befe950e4c074302aa885ed1213c5459 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/befe950e4c074302aa885ed1213c5459 2024-11-20T16:53:57,053 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/befe950e4c074302aa885ed1213c5459, entries=150, sequenceid=304, filesize=30.5 K 2024-11-20T16:53:57,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/e87c8e48e574452889c0baaa05a27b2d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/e87c8e48e574452889c0baaa05a27b2d 2024-11-20T16:53:57,061 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/e87c8e48e574452889c0baaa05a27b2d, entries=150, sequenceid=304, filesize=12.0 K 2024-11-20T16:53:57,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/157c2172a1fd48b9a44ec676c682ddde as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/157c2172a1fd48b9a44ec676c682ddde 2024-11-20T16:53:57,068 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/157c2172a1fd48b9a44ec676c682ddde, entries=150, sequenceid=304, filesize=12.0 K 2024-11-20T16:53:57,069 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5cb6af7207e610b3daed9e5466ca8411 in 875ms, sequenceid=304, compaction requested=false 2024-11-20T16:53:57,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:57,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:57,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-20T16:53:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-20T16:53:57,072 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-20T16:53:57,072 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1820 sec 2024-11-20T16:53:57,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.1850 sec 2024-11-20T16:53:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:57,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T16:53:57,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:53:57,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:57,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:53:57,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:57,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:53:57,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:57,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ba67a62d152342d8a994e6df880c6a40_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121636547/Put/seqid=0 2024-11-20T16:53:57,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742281_1457 (size=14994) 2024-11-20T16:53:57,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121697323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:57,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121697435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:57,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:57,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:57,644 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:57,648 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ba67a62d152342d8a994e6df880c6a40_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ba67a62d152342d8a994e6df880c6a40_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:57,649 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/1f987e0c44f043d483222f4ad16908cd, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:57,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/1f987e0c44f043d483222f4ad16908cd is 175, key is test_row_0/A:col10/1732121636547/Put/seqid=0 2024-11-20T16:53:57,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742282_1458 (size=39949) 2024-11-20T16:53:57,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121697656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:57,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:57,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:57,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:57,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:57,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121697966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T16:53:58,036 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:53:58,036 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:53:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-20T16:53:58,038 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:53:58,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T16:53:58,038 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:53:58,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:53:58,053 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/1f987e0c44f043d483222f4ad16908cd 2024-11-20T16:53:58,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/4f871095ab844b01a37090eef1dbbbb8 is 50, key is test_row_0/B:col10/1732121636547/Put/seqid=0 2024-11-20T16:53:58,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742283_1459 (size=12301) 2024-11-20T16:53:58,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T16:53:58,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T16:53:58,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:58,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:58,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:58,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:58,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:58,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T16:53:58,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:58,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:58,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:58,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:58,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T16:53:58,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/4f871095ab844b01a37090eef1dbbbb8 2024-11-20T16:53:58,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/b3948227f28a499a829762816ef256d0 is 50, key is test_row_0/C:col10/1732121636547/Put/seqid=0 2024-11-20T16:53:58,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742284_1460 (size=12301) 2024-11-20T16:53:58,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121698486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:58,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T16:53:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:58,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:58,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:58,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T16:53:58,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:58,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:58,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:58,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:58,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:58,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T16:53:58,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T16:53:58,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:58,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:58,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:58,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:58,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:53:58,803 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33377 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b60eac9bda43,33929,1732121526384, table=TestAcidGuarantees, region=5cb6af7207e610b3daed9e5466ca8411. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T16:53:58,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/b3948227f28a499a829762816ef256d0 2024-11-20T16:53:58,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/1f987e0c44f043d483222f4ad16908cd as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1f987e0c44f043d483222f4ad16908cd 2024-11-20T16:53:58,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1f987e0c44f043d483222f4ad16908cd, entries=200, sequenceid=320, filesize=39.0 K 2024-11-20T16:53:58,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/4f871095ab844b01a37090eef1dbbbb8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/4f871095ab844b01a37090eef1dbbbb8 2024-11-20T16:53:58,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/4f871095ab844b01a37090eef1dbbbb8, entries=150, sequenceid=320, filesize=12.0 K 2024-11-20T16:53:58,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/b3948227f28a499a829762816ef256d0 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/b3948227f28a499a829762816ef256d0 2024-11-20T16:53:58,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/b3948227f28a499a829762816ef256d0, entries=150, sequenceid=320, filesize=12.0 K 2024-11-20T16:53:58,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5cb6af7207e610b3daed9e5466ca8411 in 1662ms, sequenceid=320, compaction requested=true 2024-11-20T16:53:58,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:58,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:53:58,892 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:58,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:58,892 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:58,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:53:58,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:58,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:53:58,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:58,894 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:58,894 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/A is initiating minor compaction (all files) 2024-11-20T16:53:58,894 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/A in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:58,894 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1ec13824234e45798a49657a45ef1716, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/befe950e4c074302aa885ed1213c5459, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1f987e0c44f043d483222f4ad16908cd] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=100.7 K 2024-11-20T16:53:58,894 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:58,894 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1ec13824234e45798a49657a45ef1716, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/befe950e4c074302aa885ed1213c5459, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1f987e0c44f043d483222f4ad16908cd] 2024-11-20T16:53:58,894 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:58,894 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/B is initiating minor compaction (all files) 2024-11-20T16:53:58,894 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/B in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:58,894 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/b3f3731f53c7425099a96cf17e4d0f18, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/e87c8e48e574452889c0baaa05a27b2d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/4f871095ab844b01a37090eef1dbbbb8] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=36.7 K 2024-11-20T16:53:58,894 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ec13824234e45798a49657a45ef1716, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732121633994 2024-11-20T16:53:58,895 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting b3f3731f53c7425099a96cf17e4d0f18, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732121633994 2024-11-20T16:53:58,895 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting befe950e4c074302aa885ed1213c5459, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732121635322 2024-11-20T16:53:58,895 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting e87c8e48e574452889c0baaa05a27b2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732121635322 2024-11-20T16:53:58,895 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f987e0c44f043d483222f4ad16908cd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732121636547 2024-11-20T16:53:58,895 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f871095ab844b01a37090eef1dbbbb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732121636547 2024-11-20T16:53:58,906 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#B#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:58,907 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/363c0790ba024ae7b862547bb001c585 is 50, key is test_row_0/B:col10/1732121636547/Put/seqid=0 2024-11-20T16:53:58,908 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:58,916 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120422a910cefa446f6a1ac209b43794086_5cb6af7207e610b3daed9e5466ca8411 store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:58,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742285_1461 (size=13051) 2024-11-20T16:53:58,918 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120422a910cefa446f6a1ac209b43794086_5cb6af7207e610b3daed9e5466ca8411, store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:58,919 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120422a910cefa446f6a1ac209b43794086_5cb6af7207e610b3daed9e5466ca8411 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:58,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742286_1462 (size=4469) 2024-11-20T16:53:58,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T16:53:58,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:58,954 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T16:53:58,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:53:58,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:58,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:53:58,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:58,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:53:58,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:53:58,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208fd06698ab4948598d476b8aaec1fcc6_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121637292/Put/seqid=0 2024-11-20T16:53:58,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742287_1463 (size=12454) 2024-11-20T16:53:59,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T16:53:59,322 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/363c0790ba024ae7b862547bb001c585 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/363c0790ba024ae7b862547bb001c585 2024-11-20T16:53:59,325 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/B of 5cb6af7207e610b3daed9e5466ca8411 into 363c0790ba024ae7b862547bb001c585(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:59,325 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:59,325 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/B, priority=13, startTime=1732121638892; duration=0sec 2024-11-20T16:53:59,325 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:53:59,325 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:B 2024-11-20T16:53:59,325 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:53:59,326 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:53:59,326 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/C is initiating minor compaction (all files) 2024-11-20T16:53:59,326 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/C in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:53:59,326 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/ae5a2364ded14e9b952cd79391d13011, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/157c2172a1fd48b9a44ec676c682ddde, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/b3948227f28a499a829762816ef256d0] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=36.7 K 2024-11-20T16:53:59,326 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting ae5a2364ded14e9b952cd79391d13011, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732121633994 2024-11-20T16:53:59,327 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 157c2172a1fd48b9a44ec676c682ddde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732121635322 2024-11-20T16:53:59,327 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting b3948227f28a499a829762816ef256d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732121636547 2024-11-20T16:53:59,328 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#A#compaction#383 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:59,329 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/77a26497fc764475aae08d344f06d1be is 175, key is test_row_0/A:col10/1732121636547/Put/seqid=0 2024-11-20T16:53:59,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742288_1464 (size=32005) 2024-11-20T16:53:59,335 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#C#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:53:59,337 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/5ee8e71cd979483297e6fb48e276bfe4 is 50, key is test_row_0/C:col10/1732121636547/Put/seqid=0 2024-11-20T16:53:59,347 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/77a26497fc764475aae08d344f06d1be as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/77a26497fc764475aae08d344f06d1be 2024-11-20T16:53:59,356 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/A of 5cb6af7207e610b3daed9e5466ca8411 into 77a26497fc764475aae08d344f06d1be(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:59,356 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:59,356 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/A, priority=13, startTime=1732121638892; duration=0sec 2024-11-20T16:53:59,356 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:59,356 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:A 2024-11-20T16:53:59,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742289_1465 (size=13051) 2024-11-20T16:53:59,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:53:59,369 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208fd06698ab4948598d476b8aaec1fcc6_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208fd06698ab4948598d476b8aaec1fcc6_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:59,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/35e9c4f9543e4990bdd4b8ea7c01b5d7, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:53:59,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/35e9c4f9543e4990bdd4b8ea7c01b5d7 is 175, key is test_row_0/A:col10/1732121637292/Put/seqid=0 2024-11-20T16:53:59,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742290_1466 (size=31255) 2024-11-20T16:53:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:53:59,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. as already flushing 2024-11-20T16:53:59,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121699566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:59,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:59,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:59,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:59,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:59,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121699676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:59,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:59,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:59,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:53:59,763 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/5ee8e71cd979483297e6fb48e276bfe4 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/5ee8e71cd979483297e6fb48e276bfe4 2024-11-20T16:53:59,767 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/C of 5cb6af7207e610b3daed9e5466ca8411 into 5ee8e71cd979483297e6fb48e276bfe4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:53:59,767 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:53:59,767 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/C, priority=13, startTime=1732121638893; duration=0sec 2024-11-20T16:53:59,767 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:53:59,767 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:C 2024-11-20T16:53:59,776 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/35e9c4f9543e4990bdd4b8ea7c01b5d7 2024-11-20T16:53:59,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/d67d4b8697f64a138ab5ae9cbd918ec1 is 50, key is test_row_0/B:col10/1732121637292/Put/seqid=0 2024-11-20T16:53:59,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742291_1467 (size=12301) 2024-11-20T16:53:59,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:53:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121699895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:53:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:53:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T16:54:00,196 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/d67d4b8697f64a138ab5ae9cbd918ec1 2024-11-20T16:54:00,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/c268c276cb194af5ab0d369c29df9aae is 50, key is test_row_0/C:col10/1732121637292/Put/seqid=0 2024-11-20T16:54:00,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121700216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:00,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742292_1468 (size=12301) 2024-11-20T16:54:00,227 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/c268c276cb194af5ab0d369c29df9aae 2024-11-20T16:54:00,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/35e9c4f9543e4990bdd4b8ea7c01b5d7 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/35e9c4f9543e4990bdd4b8ea7c01b5d7 2024-11-20T16:54:00,239 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/35e9c4f9543e4990bdd4b8ea7c01b5d7, entries=150, sequenceid=339, filesize=30.5 K 2024-11-20T16:54:00,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/d67d4b8697f64a138ab5ae9cbd918ec1 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/d67d4b8697f64a138ab5ae9cbd918ec1 2024-11-20T16:54:00,248 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/d67d4b8697f64a138ab5ae9cbd918ec1, entries=150, sequenceid=339, filesize=12.0 K 2024-11-20T16:54:00,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/c268c276cb194af5ab0d369c29df9aae as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/c268c276cb194af5ab0d369c29df9aae 2024-11-20T16:54:00,265 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/c268c276cb194af5ab0d369c29df9aae, entries=150, sequenceid=339, filesize=12.0 K 2024-11-20T16:54:00,266 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5cb6af7207e610b3daed9e5466ca8411 in 1312ms, sequenceid=339, compaction requested=false 2024-11-20T16:54:00,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:54:00,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:54:00,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-20T16:54:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-20T16:54:00,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-20T16:54:00,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2300 sec 2024-11-20T16:54:00,279 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.2340 sec 2024-11-20T16:54:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:00,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T16:54:00,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:54:00,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:00,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:54:00,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:00,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:54:00,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:00,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209e3cd15d1d884b51a207d9286a4cca2d_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121640731/Put/seqid=0 2024-11-20T16:54:00,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742293_1469 (size=14994) 2024-11-20T16:54:00,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:00,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121700848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:00,877 INFO [Thread-1688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1688. 2024-11-20T16:54:00,877 DEBUG [Thread-1688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:00,877 DEBUG [Thread-1688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:00,877 DEBUG [Thread-1688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:00,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:00,885 INFO [Thread-1686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1686. 2024-11-20T16:54:00,885 DEBUG [Thread-1686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:00,885 DEBUG [Thread-1686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:00,885 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:00,885 DEBUG [Thread-1686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:00,887 INFO [Thread-1690 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1690. 2024-11-20T16:54:00,887 DEBUG [Thread-1690 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:00,887 DEBUG [Thread-1690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:00,887 DEBUG [Thread-1690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:00,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:00,890 INFO [Thread-1684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1684. 2024-11-20T16:54:00,890 DEBUG [Thread-1684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:00,890 DEBUG [Thread-1684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:00,890 DEBUG [Thread-1684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:00,890 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:00,896 INFO [Thread-1692 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1692. 2024-11-20T16:54:00,896 DEBUG [Thread-1692 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:00,896 DEBUG [Thread-1692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:00,896 DEBUG [Thread-1692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:00,896 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:00,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121700966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:00,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:00,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:00,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:01,144 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:01,148 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209e3cd15d1d884b51a207d9286a4cca2d_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209e3cd15d1d884b51a207d9286a4cca2d_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:01,149 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/48716264354e4230919ea91c9c139222, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:54:01,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/48716264354e4230919ea91c9c139222 is 175, key is test_row_0/A:col10/1732121640731/Put/seqid=0 2024-11-20T16:54:01,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742294_1470 (size=39949) 2024-11-20T16:54:01,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121701176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:01,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:01,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:01,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:01,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121701486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:01,553 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/48716264354e4230919ea91c9c139222 2024-11-20T16:54:01,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/d57d262c6b364de382d3c32ee706f07f is 50, key is test_row_0/B:col10/1732121640731/Put/seqid=0 2024-11-20T16:54:01,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742295_1471 (size=12301) 2024-11-20T16:54:01,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:01,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49410 deadline: 1732121701946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732121701946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:01,947 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 5cb6af7207e610b3daed9e5466ca8411 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18307 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:01,947 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 5cb6af7207e610b3daed9e5466ca8411 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18297 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:01,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/d57d262c6b364de382d3c32ee706f07f 2024-11-20T16:54:01,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/a4a06f1bf28f4a11a8f10fe42adfbc74 is 50, key is test_row_0/C:col10/1732121640731/Put/seqid=0 2024-11-20T16:54:01,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742296_1472 (size=12301) 2024-11-20T16:54:01,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49412 deadline: 1732121701986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:01,986 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 5cb6af7207e610b3daed9e5466ca8411 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18336 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:01,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:01,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:01,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:01,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49382 deadline: 1732121701996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:02,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49426 deadline: 1732121702036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:02,037 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 5cb6af7207e610b3daed9e5466ca8411 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18386 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:02,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:02,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:02,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T16:54:02,206 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:02,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/a4a06f1bf28f4a11a8f10fe42adfbc74 2024-11-20T16:54:02,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/48716264354e4230919ea91c9c139222 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/48716264354e4230919ea91c9c139222 2024-11-20T16:54:02,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/48716264354e4230919ea91c9c139222, entries=200, sequenceid=356, filesize=39.0 K 2024-11-20T16:54:02,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/d57d262c6b364de382d3c32ee706f07f as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/d57d262c6b364de382d3c32ee706f07f 2024-11-20T16:54:02,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/d57d262c6b364de382d3c32ee706f07f, entries=150, sequenceid=356, filesize=12.0 K 2024-11-20T16:54:02,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/a4a06f1bf28f4a11a8f10fe42adfbc74 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/a4a06f1bf28f4a11a8f10fe42adfbc74 2024-11-20T16:54:02,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/a4a06f1bf28f4a11a8f10fe42adfbc74, entries=150, sequenceid=356, filesize=12.0 K 2024-11-20T16:54:02,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5cb6af7207e610b3daed9e5466ca8411 in 1657ms, sequenceid=356, compaction requested=true 2024-11-20T16:54:02,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:54:02,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:54:02,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:02,389 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:54:02,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:54:02,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:02,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5cb6af7207e610b3daed9e5466ca8411:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:54:02,389 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:54:02,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:02,390 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:54:02,390 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:54:02,390 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/A is initiating minor compaction (all files) 2024-11-20T16:54:02,390 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/B is initiating minor compaction (all files) 2024-11-20T16:54:02,390 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/B in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:54:02,390 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/A in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:54:02,390 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/363c0790ba024ae7b862547bb001c585, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/d67d4b8697f64a138ab5ae9cbd918ec1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/d57d262c6b364de382d3c32ee706f07f] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=36.8 K 2024-11-20T16:54:02,390 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/77a26497fc764475aae08d344f06d1be, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/35e9c4f9543e4990bdd4b8ea7c01b5d7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/48716264354e4230919ea91c9c139222] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=100.8 K 2024-11-20T16:54:02,390 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:54:02,390 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/77a26497fc764475aae08d344f06d1be, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/35e9c4f9543e4990bdd4b8ea7c01b5d7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/48716264354e4230919ea91c9c139222] 2024-11-20T16:54:02,390 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 363c0790ba024ae7b862547bb001c585, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732121636547 2024-11-20T16:54:02,390 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77a26497fc764475aae08d344f06d1be, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732121636547 2024-11-20T16:54:02,391 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting d67d4b8697f64a138ab5ae9cbd918ec1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732121637292 2024-11-20T16:54:02,391 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35e9c4f9543e4990bdd4b8ea7c01b5d7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732121637292 2024-11-20T16:54:02,395 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48716264354e4230919ea91c9c139222, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732121639552 2024-11-20T16:54:02,395 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting d57d262c6b364de382d3c32ee706f07f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732121639552 2024-11-20T16:54:02,400 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:54:02,402 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411209268018b87564810a7a13a0b4d66828d_5cb6af7207e610b3daed9e5466ca8411 store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:54:02,402 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#B#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:02,402 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/651d916c30c84723abe7d24d187fbd67 is 50, key is test_row_0/B:col10/1732121640731/Put/seqid=0 2024-11-20T16:54:02,408 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411209268018b87564810a7a13a0b4d66828d_5cb6af7207e610b3daed9e5466ca8411, store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:54:02,408 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209268018b87564810a7a13a0b4d66828d_5cb6af7207e610b3daed9e5466ca8411 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:54:02,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742297_1473 (size=13153) 2024-11-20T16:54:02,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742298_1474 (size=4469) 2024-11-20T16:54:02,429 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#A#compaction#392 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:02,429 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/07aea4b18cc84f3aa77aa48a8a9cd511 is 175, key is test_row_0/A:col10/1732121640731/Put/seqid=0 2024-11-20T16:54:02,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742299_1475 (size=32107) 2024-11-20T16:54:02,820 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/651d916c30c84723abe7d24d187fbd67 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/651d916c30c84723abe7d24d187fbd67 2024-11-20T16:54:02,823 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/B of 5cb6af7207e610b3daed9e5466ca8411 into 651d916c30c84723abe7d24d187fbd67(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:02,823 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:54:02,823 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/B, priority=13, startTime=1732121642389; duration=0sec 2024-11-20T16:54:02,823 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:02,824 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:B 2024-11-20T16:54:02,824 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:54:02,824 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:54:02,824 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 5cb6af7207e610b3daed9e5466ca8411/C is initiating minor compaction (all files) 2024-11-20T16:54:02,824 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5cb6af7207e610b3daed9e5466ca8411/C in TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:54:02,825 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/5ee8e71cd979483297e6fb48e276bfe4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/c268c276cb194af5ab0d369c29df9aae, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/a4a06f1bf28f4a11a8f10fe42adfbc74] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp, totalSize=36.8 K 2024-11-20T16:54:02,825 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ee8e71cd979483297e6fb48e276bfe4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732121636547 2024-11-20T16:54:02,825 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting c268c276cb194af5ab0d369c29df9aae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732121637292 2024-11-20T16:54:02,825 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting a4a06f1bf28f4a11a8f10fe42adfbc74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732121639552 2024-11-20T16:54:02,831 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5cb6af7207e610b3daed9e5466ca8411#C#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:02,831 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/05db51eea44943839012074ccec17190 is 50, key is test_row_0/C:col10/1732121640731/Put/seqid=0 2024-11-20T16:54:02,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742300_1476 (size=13153) 2024-11-20T16:54:02,837 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/07aea4b18cc84f3aa77aa48a8a9cd511 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/07aea4b18cc84f3aa77aa48a8a9cd511 2024-11-20T16:54:02,841 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/A of 5cb6af7207e610b3daed9e5466ca8411 into 07aea4b18cc84f3aa77aa48a8a9cd511(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:02,841 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:54:02,841 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/A, priority=13, startTime=1732121642389; duration=0sec 2024-11-20T16:54:02,841 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:02,841 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:A 2024-11-20T16:54:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:03,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T16:54:03,010 INFO [Thread-1673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1673. 2024-11-20T16:54:03,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:54:03,011 DEBUG [Thread-1673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:03,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:03,011 DEBUG [Thread-1673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:03,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:54:03,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:03,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:54:03,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:03,011 DEBUG [Thread-1673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:03,011 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:03,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200226ef310fd34b79b19cd99d51c4717a_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_0/A:col10/1732121640847/Put/seqid=0 2024-11-20T16:54:03,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742301_1477 (size=12454) 2024-11-20T16:54:03,240 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/05db51eea44943839012074ccec17190 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/05db51eea44943839012074ccec17190 2024-11-20T16:54:03,244 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5cb6af7207e610b3daed9e5466ca8411/C of 5cb6af7207e610b3daed9e5466ca8411 into 05db51eea44943839012074ccec17190(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:03,244 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:54:03,244 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., storeName=5cb6af7207e610b3daed9e5466ca8411/C, priority=13, startTime=1732121642389; duration=0sec 2024-11-20T16:54:03,244 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:03,244 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5cb6af7207e610b3daed9e5466ca8411:C 2024-11-20T16:54:03,420 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:03,423 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200226ef310fd34b79b19cd99d51c4717a_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200226ef310fd34b79b19cd99d51c4717a_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:03,424 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/0098e84e65824a87b2e0c5453c9a8910, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:54:03,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/0098e84e65824a87b2e0c5453c9a8910 is 175, key is test_row_0/A:col10/1732121640847/Put/seqid=0 2024-11-20T16:54:03,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742302_1478 (size=31255) 2024-11-20T16:54:03,828 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=377, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/0098e84e65824a87b2e0c5453c9a8910 2024-11-20T16:54:03,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/05ab2cd90b394957a1199b6051cd3e9b is 50, key is test_row_0/B:col10/1732121640847/Put/seqid=0 2024-11-20T16:54:03,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742303_1479 (size=12301) 2024-11-20T16:54:04,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/05ab2cd90b394957a1199b6051cd3e9b 2024-11-20T16:54:04,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/4e492f6ab8fa458b88f6e526a207879e is 50, key is test_row_0/C:col10/1732121640847/Put/seqid=0 2024-11-20T16:54:04,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742304_1480 (size=12301) 2024-11-20T16:54:04,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/4e492f6ab8fa458b88f6e526a207879e 2024-11-20T16:54:04,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/0098e84e65824a87b2e0c5453c9a8910 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0098e84e65824a87b2e0c5453c9a8910 2024-11-20T16:54:04,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0098e84e65824a87b2e0c5453c9a8910, entries=150, sequenceid=377, filesize=30.5 K 2024-11-20T16:54:04,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/05ab2cd90b394957a1199b6051cd3e9b as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/05ab2cd90b394957a1199b6051cd3e9b 2024-11-20T16:54:04,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/05ab2cd90b394957a1199b6051cd3e9b, entries=150, sequenceid=377, filesize=12.0 K 2024-11-20T16:54:04,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/4e492f6ab8fa458b88f6e526a207879e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/4e492f6ab8fa458b88f6e526a207879e 2024-11-20T16:54:04,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/4e492f6ab8fa458b88f6e526a207879e, entries=150, sequenceid=377, filesize=12.0 K 2024-11-20T16:54:04,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 5cb6af7207e610b3daed9e5466ca8411 in 1651ms, sequenceid=377, compaction requested=false 2024-11-20T16:54:04,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5cb6af7207e610b3daed9e5466ca8411: 2024-11-20T16:54:04,673 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T16:54:09,316 WARN [regionserver/b60eac9bda43:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-20T16:54:09,319 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0eb90718ee0f4de894bb8a470cf3a573, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c5215b51e51a40da8f0803585331daeb, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/59e46a02378a4b9194cf71dc79b4ce33, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/e6e6154c27b44820b3b7677e326bc8e0, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/6139bcb0689d46bfa0dc1575d868d110, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/9d268cc08bc44f3a950a24aaadd09c00, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/aab16f6655104bf5a8f9fb42e2344857, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0d6b5ea88b58410f93bdcc8c4eed501c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0f6a42e733864a4ea53cbb706be237be, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/ebedef411fa34465b3d9aabc99a240d9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/fe9c767ae28e4635bcad8ad7df4be6a2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c88946248c5241029c108fc4e881e43a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c5adb3b84f4541dda19e287a5478cf90, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/d9648ba0514442b9841f2d92db3ce7cd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/ec764d8478dc40c3908e2a74b253fc24, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1ea4663de0b04135ad6c0842a259150e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/6047c26f6f2044bf9835e2f2da6fb11b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/9ad804bd1fb8435d8d77a2979a18e04d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/b98f1409dc144293b661f07432ea2805, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1ec13824234e45798a49657a45ef1716, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/befe950e4c074302aa885ed1213c5459, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1f987e0c44f043d483222f4ad16908cd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/77a26497fc764475aae08d344f06d1be, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/35e9c4f9543e4990bdd4b8ea7c01b5d7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/48716264354e4230919ea91c9c139222] to archive 2024-11-20T16:54:09,320 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T16:54:09,321 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0eb90718ee0f4de894bb8a470cf3a573 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0eb90718ee0f4de894bb8a470cf3a573 2024-11-20T16:54:09,322 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c5215b51e51a40da8f0803585331daeb to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c5215b51e51a40da8f0803585331daeb 2024-11-20T16:54:09,323 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/59e46a02378a4b9194cf71dc79b4ce33 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/59e46a02378a4b9194cf71dc79b4ce33 2024-11-20T16:54:09,323 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/e6e6154c27b44820b3b7677e326bc8e0 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/e6e6154c27b44820b3b7677e326bc8e0 2024-11-20T16:54:09,324 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/6139bcb0689d46bfa0dc1575d868d110 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/6139bcb0689d46bfa0dc1575d868d110 2024-11-20T16:54:09,325 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/9d268cc08bc44f3a950a24aaadd09c00 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/9d268cc08bc44f3a950a24aaadd09c00 2024-11-20T16:54:09,326 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/aab16f6655104bf5a8f9fb42e2344857 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/aab16f6655104bf5a8f9fb42e2344857 2024-11-20T16:54:09,326 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0d6b5ea88b58410f93bdcc8c4eed501c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0d6b5ea88b58410f93bdcc8c4eed501c 2024-11-20T16:54:09,327 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0f6a42e733864a4ea53cbb706be237be to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0f6a42e733864a4ea53cbb706be237be 2024-11-20T16:54:09,328 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/ebedef411fa34465b3d9aabc99a240d9 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/ebedef411fa34465b3d9aabc99a240d9 2024-11-20T16:54:09,329 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/fe9c767ae28e4635bcad8ad7df4be6a2 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/fe9c767ae28e4635bcad8ad7df4be6a2 2024-11-20T16:54:09,330 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c88946248c5241029c108fc4e881e43a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c88946248c5241029c108fc4e881e43a 2024-11-20T16:54:09,330 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c5adb3b84f4541dda19e287a5478cf90 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c5adb3b84f4541dda19e287a5478cf90 2024-11-20T16:54:09,331 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/d9648ba0514442b9841f2d92db3ce7cd to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/d9648ba0514442b9841f2d92db3ce7cd 2024-11-20T16:54:09,332 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/ec764d8478dc40c3908e2a74b253fc24 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/ec764d8478dc40c3908e2a74b253fc24 2024-11-20T16:54:09,333 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1ea4663de0b04135ad6c0842a259150e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1ea4663de0b04135ad6c0842a259150e 2024-11-20T16:54:09,334 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/6047c26f6f2044bf9835e2f2da6fb11b to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/6047c26f6f2044bf9835e2f2da6fb11b 2024-11-20T16:54:09,335 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/9ad804bd1fb8435d8d77a2979a18e04d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/9ad804bd1fb8435d8d77a2979a18e04d 2024-11-20T16:54:09,336 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/b98f1409dc144293b661f07432ea2805 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/b98f1409dc144293b661f07432ea2805 2024-11-20T16:54:09,337 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1ec13824234e45798a49657a45ef1716 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1ec13824234e45798a49657a45ef1716 2024-11-20T16:54:09,338 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/befe950e4c074302aa885ed1213c5459 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/befe950e4c074302aa885ed1213c5459 2024-11-20T16:54:09,339 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1f987e0c44f043d483222f4ad16908cd to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/1f987e0c44f043d483222f4ad16908cd 2024-11-20T16:54:09,340 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/77a26497fc764475aae08d344f06d1be to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/77a26497fc764475aae08d344f06d1be 2024-11-20T16:54:09,340 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/35e9c4f9543e4990bdd4b8ea7c01b5d7 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/35e9c4f9543e4990bdd4b8ea7c01b5d7 2024-11-20T16:54:09,341 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/48716264354e4230919ea91c9c139222 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/48716264354e4230919ea91c9c139222 2024-11-20T16:54:09,346 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/b5e03b3e93fe443d8826ba67f5eed5f8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/5e466f09559e4953940c62aba1f649b9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/f969fc95faa240fcbdb9f291b087d00d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/c5f5b9bfaaa94b7d8676611b5b58d1f8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/a851663a9a4b4cc1850235b969292fe5, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/526ebd707c204a29bb44cb078075c1b8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/966aec3242bb4ccda4dc667eb9f3467e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/c902ec8d26cd4d22b94c0fa3b070ccd0, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/c8c9b6e46380479aac256f05f0a6197a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/44f0d1f018614b99a2ee5815fb2a9ce5, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/1b6256993eda4339b90a800a883bef83, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/b1ad37d21a534ecaa60978128ba6cd46, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/e7e543333c7b4b9cb5a2ec1755dd4157, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/6ad2ec633c7946fc8aaffa15bc01ccf3, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/d327fe367f6d4263aa7d71d9219385bb, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/4502c1fd47894a608722355df5ea1bd8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/7bd61a8bc5d4463da93fe5089dc47154, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/1f58de598a7e4fbf9c05be1ffa6756c4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/b3f3731f53c7425099a96cf17e4d0f18, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/5ee24590dcf34c33a42ae22605cd4d56, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/e87c8e48e574452889c0baaa05a27b2d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/363c0790ba024ae7b862547bb001c585, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/4f871095ab844b01a37090eef1dbbbb8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/d67d4b8697f64a138ab5ae9cbd918ec1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/d57d262c6b364de382d3c32ee706f07f] to archive 2024-11-20T16:54:09,347 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T16:54:09,349 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/b5e03b3e93fe443d8826ba67f5eed5f8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/b5e03b3e93fe443d8826ba67f5eed5f8 2024-11-20T16:54:09,350 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/5e466f09559e4953940c62aba1f649b9 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/5e466f09559e4953940c62aba1f649b9 2024-11-20T16:54:09,351 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/f969fc95faa240fcbdb9f291b087d00d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/f969fc95faa240fcbdb9f291b087d00d 2024-11-20T16:54:09,352 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/c5f5b9bfaaa94b7d8676611b5b58d1f8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/c5f5b9bfaaa94b7d8676611b5b58d1f8 2024-11-20T16:54:09,353 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/a851663a9a4b4cc1850235b969292fe5 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/a851663a9a4b4cc1850235b969292fe5 2024-11-20T16:54:09,354 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/526ebd707c204a29bb44cb078075c1b8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/526ebd707c204a29bb44cb078075c1b8 2024-11-20T16:54:09,355 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/966aec3242bb4ccda4dc667eb9f3467e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/966aec3242bb4ccda4dc667eb9f3467e 2024-11-20T16:54:09,356 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/c902ec8d26cd4d22b94c0fa3b070ccd0 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/c902ec8d26cd4d22b94c0fa3b070ccd0 2024-11-20T16:54:09,357 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/c8c9b6e46380479aac256f05f0a6197a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/c8c9b6e46380479aac256f05f0a6197a 2024-11-20T16:54:09,359 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/44f0d1f018614b99a2ee5815fb2a9ce5 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/44f0d1f018614b99a2ee5815fb2a9ce5 2024-11-20T16:54:09,360 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/1b6256993eda4339b90a800a883bef83 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/1b6256993eda4339b90a800a883bef83 2024-11-20T16:54:09,361 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/b1ad37d21a534ecaa60978128ba6cd46 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/b1ad37d21a534ecaa60978128ba6cd46 2024-11-20T16:54:09,362 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/e7e543333c7b4b9cb5a2ec1755dd4157 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/e7e543333c7b4b9cb5a2ec1755dd4157 2024-11-20T16:54:09,363 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/6ad2ec633c7946fc8aaffa15bc01ccf3 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/6ad2ec633c7946fc8aaffa15bc01ccf3 2024-11-20T16:54:09,364 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/d327fe367f6d4263aa7d71d9219385bb to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/d327fe367f6d4263aa7d71d9219385bb 2024-11-20T16:54:09,365 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/4502c1fd47894a608722355df5ea1bd8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/4502c1fd47894a608722355df5ea1bd8 2024-11-20T16:54:09,366 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/7bd61a8bc5d4463da93fe5089dc47154 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/7bd61a8bc5d4463da93fe5089dc47154 2024-11-20T16:54:09,367 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/1f58de598a7e4fbf9c05be1ffa6756c4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/1f58de598a7e4fbf9c05be1ffa6756c4 2024-11-20T16:54:09,368 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/b3f3731f53c7425099a96cf17e4d0f18 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/b3f3731f53c7425099a96cf17e4d0f18 2024-11-20T16:54:09,370 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/5ee24590dcf34c33a42ae22605cd4d56 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/5ee24590dcf34c33a42ae22605cd4d56 2024-11-20T16:54:09,371 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/e87c8e48e574452889c0baaa05a27b2d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/e87c8e48e574452889c0baaa05a27b2d 2024-11-20T16:54:09,373 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/363c0790ba024ae7b862547bb001c585 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/363c0790ba024ae7b862547bb001c585 2024-11-20T16:54:09,374 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/4f871095ab844b01a37090eef1dbbbb8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/4f871095ab844b01a37090eef1dbbbb8 2024-11-20T16:54:09,375 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/d67d4b8697f64a138ab5ae9cbd918ec1 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/d67d4b8697f64a138ab5ae9cbd918ec1 2024-11-20T16:54:09,376 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/d57d262c6b364de382d3c32ee706f07f to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/d57d262c6b364de382d3c32ee706f07f 2024-11-20T16:54:09,381 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/0d3dc930d74a463895e1f4e638445394, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/473b0fe3f6c94429b19a71a6d27d4503, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/e95ab6e3f36e483a8b680d4238c4e070, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/6be37acc8e0e477481f9e2395b97a9e3, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/fc3993220a8c464ca4a1f7cea64c8aff, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/724f293805f84cf5986235f7c8a33b61, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/2f9575d1e7a14d5e87c3a7fc75a55aae, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/2752c6f2420a4905af7c5eb47354a2d2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/974df1258f14436b8a6ea5240de0e274, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/047deefeffda4bd0a57841577021c16a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/cbbcfed0db0147b0968d9910089ab0f6, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/4a10f493777843ef92902b6946b037b8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/1e691ca32de84a1b9d208577adde8498, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/6cd2f4fb7ab34d249ea1975c5d69a98d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/f913ab27ef54489998a526f312770532, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/138aa0407c704eeaa4f81b93df37f673, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/b18252c2852847f69df201b92db8a22a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/01e2a5d420504680a994c38a4b190af9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/ae5a2364ded14e9b952cd79391d13011, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/ce7ba37e1912404d81e87359892efb12, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/157c2172a1fd48b9a44ec676c682ddde, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/5ee8e71cd979483297e6fb48e276bfe4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/b3948227f28a499a829762816ef256d0, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/c268c276cb194af5ab0d369c29df9aae, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/a4a06f1bf28f4a11a8f10fe42adfbc74] to archive 2024-11-20T16:54:09,383 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T16:54:09,384 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/0d3dc930d74a463895e1f4e638445394 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/0d3dc930d74a463895e1f4e638445394 2024-11-20T16:54:09,385 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/473b0fe3f6c94429b19a71a6d27d4503 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/473b0fe3f6c94429b19a71a6d27d4503 2024-11-20T16:54:09,387 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/e95ab6e3f36e483a8b680d4238c4e070 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/e95ab6e3f36e483a8b680d4238c4e070 2024-11-20T16:54:09,388 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/6be37acc8e0e477481f9e2395b97a9e3 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/6be37acc8e0e477481f9e2395b97a9e3 2024-11-20T16:54:09,389 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/fc3993220a8c464ca4a1f7cea64c8aff to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/fc3993220a8c464ca4a1f7cea64c8aff 2024-11-20T16:54:09,390 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/724f293805f84cf5986235f7c8a33b61 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/724f293805f84cf5986235f7c8a33b61 2024-11-20T16:54:09,391 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/2f9575d1e7a14d5e87c3a7fc75a55aae to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/2f9575d1e7a14d5e87c3a7fc75a55aae 2024-11-20T16:54:09,392 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/2752c6f2420a4905af7c5eb47354a2d2 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/2752c6f2420a4905af7c5eb47354a2d2 2024-11-20T16:54:09,393 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/974df1258f14436b8a6ea5240de0e274 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/974df1258f14436b8a6ea5240de0e274 2024-11-20T16:54:09,394 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/047deefeffda4bd0a57841577021c16a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/047deefeffda4bd0a57841577021c16a 2024-11-20T16:54:09,395 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/cbbcfed0db0147b0968d9910089ab0f6 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/cbbcfed0db0147b0968d9910089ab0f6 2024-11-20T16:54:09,397 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/4a10f493777843ef92902b6946b037b8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/4a10f493777843ef92902b6946b037b8 2024-11-20T16:54:09,398 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/1e691ca32de84a1b9d208577adde8498 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/1e691ca32de84a1b9d208577adde8498 2024-11-20T16:54:09,399 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/6cd2f4fb7ab34d249ea1975c5d69a98d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/6cd2f4fb7ab34d249ea1975c5d69a98d 2024-11-20T16:54:09,400 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/f913ab27ef54489998a526f312770532 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/f913ab27ef54489998a526f312770532 2024-11-20T16:54:09,401 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/138aa0407c704eeaa4f81b93df37f673 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/138aa0407c704eeaa4f81b93df37f673 2024-11-20T16:54:09,402 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/b18252c2852847f69df201b92db8a22a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/b18252c2852847f69df201b92db8a22a 2024-11-20T16:54:09,403 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/01e2a5d420504680a994c38a4b190af9 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/01e2a5d420504680a994c38a4b190af9 2024-11-20T16:54:09,404 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/ae5a2364ded14e9b952cd79391d13011 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/ae5a2364ded14e9b952cd79391d13011 2024-11-20T16:54:09,405 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/ce7ba37e1912404d81e87359892efb12 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/ce7ba37e1912404d81e87359892efb12 2024-11-20T16:54:09,406 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/157c2172a1fd48b9a44ec676c682ddde to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/157c2172a1fd48b9a44ec676c682ddde 2024-11-20T16:54:09,407 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/5ee8e71cd979483297e6fb48e276bfe4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/5ee8e71cd979483297e6fb48e276bfe4 2024-11-20T16:54:09,408 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/b3948227f28a499a829762816ef256d0 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/b3948227f28a499a829762816ef256d0 2024-11-20T16:54:09,409 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/c268c276cb194af5ab0d369c29df9aae to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/c268c276cb194af5ab0d369c29df9aae 2024-11-20T16:54:09,409 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/a4a06f1bf28f4a11a8f10fe42adfbc74 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/a4a06f1bf28f4a11a8f10fe42adfbc74 2024-11-20T16:54:12,028 INFO [Thread-1681 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1681. 2024-11-20T16:54:12,028 DEBUG [Thread-1681 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:12,028 DEBUG [Thread-1681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:12,029 DEBUG [Thread-1681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:12,029 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:12,048 INFO [Thread-1679 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1679. 2024-11-20T16:54:12,048 INFO [Thread-1675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1675. 2024-11-20T16:54:12,048 DEBUG [Thread-1675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:12,048 DEBUG [Thread-1679 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:12,048 DEBUG [Thread-1675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:12,048 DEBUG [Thread-1679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:12,048 DEBUG [Thread-1679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:12,048 DEBUG [Thread-1675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:12,048 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:12,048 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:12,057 INFO [Thread-1677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1677. 2024-11-20T16:54:12,057 DEBUG [Thread-1677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:12,057 DEBUG [Thread-1677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:12,057 DEBUG [Thread-1677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:12,057 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:12,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T16:54:12,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 227 2024-11-20T16:54:12,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-20T16:54:12,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-20T16:54:12,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-20T16:54:12,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-20T16:54:12,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T16:54:12,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T16:54:12,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 964 2024-11-20T16:54:12,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2891 rows 2024-11-20T16:54:12,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 960 2024-11-20T16:54:12,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2880 rows 2024-11-20T16:54:12,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 953 2024-11-20T16:54:12,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2859 rows 2024-11-20T16:54:12,058 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 956 2024-11-20T16:54:12,058 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2868 rows 2024-11-20T16:54:12,058 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 954 2024-11-20T16:54:12,058 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2861 rows 2024-11-20T16:54:12,058 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T16:54:12,058 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T16:54:12,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:12,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:12,058 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:12,058 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T16:54:12,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T16:54:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T16:54:12,062 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121652061"}]},"ts":"1732121652061"} 2024-11-20T16:54:12,063 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T16:54:12,063 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T16:54:12,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T16:54:12,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5cb6af7207e610b3daed9e5466ca8411, UNASSIGN}] 2024-11-20T16:54:12,065 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5cb6af7207e610b3daed9e5466ca8411, UNASSIGN 2024-11-20T16:54:12,066 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=5cb6af7207e610b3daed9e5466ca8411, regionState=CLOSING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:12,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5cb6af7207e610b3daed9e5466ca8411, UNASSIGN because future has completed 2024-11-20T16:54:12,067 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T16:54:12,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:54:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T16:54:12,220 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(122): Close 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:12,220 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T16:54:12,220 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1722): Closing 5cb6af7207e610b3daed9e5466ca8411, disabling compactions & flushes 2024-11-20T16:54:12,220 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:54:12,220 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:54:12,220 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. after waiting 0 ms 2024-11-20T16:54:12,220 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:54:12,220 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(2902): Flushing 5cb6af7207e610b3daed9e5466ca8411 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-20T16:54:12,220 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=A 2024-11-20T16:54:12,220 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:12,220 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=B 2024-11-20T16:54:12,220 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:12,220 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5cb6af7207e610b3daed9e5466ca8411, store=C 2024-11-20T16:54:12,220 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:12,225 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208641a9c8169642fe948ba278762146e1_5cb6af7207e610b3daed9e5466ca8411 is 50, key is test_row_1/A:col10/1732121652047/Put/seqid=0 2024-11-20T16:54:12,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742305_1481 (size=9914) 2024-11-20T16:54:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T16:54:12,628 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:12,631 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208641a9c8169642fe948ba278762146e1_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208641a9c8169642fe948ba278762146e1_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:12,632 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/c72f2df1ca98493a941d874920e5a125, store: [table=TestAcidGuarantees family=A region=5cb6af7207e610b3daed9e5466ca8411] 2024-11-20T16:54:12,632 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/c72f2df1ca98493a941d874920e5a125 is 175, key is test_row_1/A:col10/1732121652047/Put/seqid=0 2024-11-20T16:54:12,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742306_1482 (size=22561) 2024-11-20T16:54:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T16:54:13,036 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=385, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/c72f2df1ca98493a941d874920e5a125 2024-11-20T16:54:13,042 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/6562bcf5383e44b8ab57050a70384954 is 50, key is test_row_1/B:col10/1732121652047/Put/seqid=0 2024-11-20T16:54:13,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742307_1483 (size=9857) 2024-11-20T16:54:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T16:54:13,446 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/6562bcf5383e44b8ab57050a70384954 2024-11-20T16:54:13,452 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/05f8942fcd2640c9bee52dfb7467c11c is 50, key is test_row_1/C:col10/1732121652047/Put/seqid=0 2024-11-20T16:54:13,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742308_1484 (size=9857) 2024-11-20T16:54:13,856 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/05f8942fcd2640c9bee52dfb7467c11c 2024-11-20T16:54:13,859 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/A/c72f2df1ca98493a941d874920e5a125 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c72f2df1ca98493a941d874920e5a125 2024-11-20T16:54:13,862 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c72f2df1ca98493a941d874920e5a125, entries=100, sequenceid=385, filesize=22.0 K 2024-11-20T16:54:13,863 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/B/6562bcf5383e44b8ab57050a70384954 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/6562bcf5383e44b8ab57050a70384954 2024-11-20T16:54:13,867 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/6562bcf5383e44b8ab57050a70384954, entries=100, sequenceid=385, filesize=9.6 K 2024-11-20T16:54:13,867 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/.tmp/C/05f8942fcd2640c9bee52dfb7467c11c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/05f8942fcd2640c9bee52dfb7467c11c 2024-11-20T16:54:13,871 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/05f8942fcd2640c9bee52dfb7467c11c, entries=100, sequenceid=385, filesize=9.6 K 2024-11-20T16:54:13,871 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 5cb6af7207e610b3daed9e5466ca8411 in 1651ms, sequenceid=385, compaction requested=true 2024-11-20T16:54:13,875 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/recovered.edits/388.seqid, newMaxSeqId=388, maxSeqId=4 2024-11-20T16:54:13,876 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. 2024-11-20T16:54:13,876 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1676): Region close journal for 5cb6af7207e610b3daed9e5466ca8411: Waiting for close lock at 1732121652220Running coprocessor pre-close hooks at 1732121652220Disabling compacts and flushes for region at 1732121652220Disabling writes for close at 1732121652220Obtaining lock to block concurrent updates at 1732121652220Preparing flush snapshotting stores in 5cb6af7207e610b3daed9e5466ca8411 at 1732121652220Finished memstore snapshotting TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732121652220Flushing stores of TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411. at 1732121652221 (+1 ms)Flushing 5cb6af7207e610b3daed9e5466ca8411/A: creating writer at 1732121652221Flushing mob file 5cb6af7207e610b3daed9e5466ca8411/A: appending metadata at 1732121652225 (+4 ms)Flushing mob file 5cb6af7207e610b3daed9e5466ca8411/A: closing flushed file at 1732121652225Flushing 5cb6af7207e610b3daed9e5466ca8411/A: appending metadata at 1732121652632 (+407 ms)Flushing 5cb6af7207e610b3daed9e5466ca8411/A: closing flushed file at 1732121652632Flushing 5cb6af7207e610b3daed9e5466ca8411/B: creating writer at 1732121653039 (+407 ms)Flushing 5cb6af7207e610b3daed9e5466ca8411/B: appending metadata at 1732121653041 (+2 ms)Flushing 5cb6af7207e610b3daed9e5466ca8411/B: closing flushed file at 1732121653041Flushing 5cb6af7207e610b3daed9e5466ca8411/C: creating writer at 1732121653449 (+408 ms)Flushing 5cb6af7207e610b3daed9e5466ca8411/C: appending metadata at 1732121653451 (+2 ms)Flushing 5cb6af7207e610b3daed9e5466ca8411/C: closing flushed file at 1732121653451Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22996b26: reopening flushed file at 1732121653859 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64a7fe17: reopening flushed file at 1732121653862 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55ea088a: reopening flushed file at 1732121653867 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 5cb6af7207e610b3daed9e5466ca8411 in 1651ms, sequenceid=385, compaction requested=true at 1732121653872 (+5 ms)Writing region close event to WAL at 1732121653873 (+1 ms)Running coprocessor post-close hooks at 1732121653876 (+3 ms)Closed at 1732121653876 2024-11-20T16:54:13,878 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(157): Closed 5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:13,878 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=5cb6af7207e610b3daed9e5466ca8411, regionState=CLOSED 2024-11-20T16:54:13,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 because future has completed 2024-11-20T16:54:13,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-20T16:54:13,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; CloseRegionProcedure 5cb6af7207e610b3daed9e5466ca8411, server=b60eac9bda43,33929,1732121526384 in 1.8130 sec 2024-11-20T16:54:13,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-20T16:54:13,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5cb6af7207e610b3daed9e5466ca8411, UNASSIGN in 1.8160 sec 2024-11-20T16:54:13,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-20T16:54:13,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8200 sec 2024-11-20T16:54:13,885 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121653885"}]},"ts":"1732121653885"} 2024-11-20T16:54:13,887 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T16:54:13,887 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T16:54:13,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8300 sec 2024-11-20T16:54:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T16:54:14,216 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:14,217 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T16:54:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:54:14,219 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:54:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T16:54:14,220 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=127, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:54:14,223 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,227 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A, FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B, FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C, FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/recovered.edits] 2024-11-20T16:54:14,231 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0098e84e65824a87b2e0c5453c9a8910 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/0098e84e65824a87b2e0c5453c9a8910 2024-11-20T16:54:14,233 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/07aea4b18cc84f3aa77aa48a8a9cd511 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/07aea4b18cc84f3aa77aa48a8a9cd511 2024-11-20T16:54:14,235 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c72f2df1ca98493a941d874920e5a125 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/A/c72f2df1ca98493a941d874920e5a125 2024-11-20T16:54:14,238 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/05ab2cd90b394957a1199b6051cd3e9b to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/05ab2cd90b394957a1199b6051cd3e9b 2024-11-20T16:54:14,239 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/651d916c30c84723abe7d24d187fbd67 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/651d916c30c84723abe7d24d187fbd67 2024-11-20T16:54:14,240 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/6562bcf5383e44b8ab57050a70384954 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/B/6562bcf5383e44b8ab57050a70384954 2024-11-20T16:54:14,243 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/05db51eea44943839012074ccec17190 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/05db51eea44943839012074ccec17190 2024-11-20T16:54:14,244 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/05f8942fcd2640c9bee52dfb7467c11c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/05f8942fcd2640c9bee52dfb7467c11c 2024-11-20T16:54:14,245 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/4e492f6ab8fa458b88f6e526a207879e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/C/4e492f6ab8fa458b88f6e526a207879e 2024-11-20T16:54:14,247 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/recovered.edits/388.seqid to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411/recovered.edits/388.seqid 2024-11-20T16:54:14,248 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,248 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T16:54:14,248 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T16:54:14,249 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T16:54:14,253 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200226ef310fd34b79b19cd99d51c4717a_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200226ef310fd34b79b19cd99d51c4717a_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,255 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112017f75dfd53074ab7bf58d691d0927746_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112017f75dfd53074ab7bf58d691d0927746_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,257 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112031c221416f784d08bce9d3754807592f_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112031c221416f784d08bce9d3754807592f_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,259 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205a7aea95d2434564996755175da72ef1_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205a7aea95d2434564996755175da72ef1_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,260 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120606f6610d8384961bd6cc9848d8bdcae_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120606f6610d8384961bd6cc9848d8bdcae_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,262 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206248512482394aedbdd4577b55eeb312_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206248512482394aedbdd4577b55eeb312_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,263 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112065f1a37be9454ca199ee63b7d1c57c87_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112065f1a37be9454ca199ee63b7d1c57c87_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,264 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112077d70475dbfb49d7ab4bfba1f824a4d4_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112077d70475dbfb49d7ab4bfba1f824a4d4_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,265 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207f96696f70de432e875f9ed602a8f208_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207f96696f70de432e875f9ed602a8f208_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,267 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207fa18d2a0ced4b899c95f86f5e6c6721_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207fa18d2a0ced4b899c95f86f5e6c6721_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,268 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208641a9c8169642fe948ba278762146e1_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208641a9c8169642fe948ba278762146e1_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,270 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208fd06698ab4948598d476b8aaec1fcc6_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208fd06698ab4948598d476b8aaec1fcc6_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,272 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112096dfb948420247a18b878b7fb6fa7e58_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112096dfb948420247a18b878b7fb6fa7e58_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,273 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209e3cd15d1d884b51a207d9286a4cca2d_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209e3cd15d1d884b51a207d9286a4cca2d_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,275 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ba67a62d152342d8a994e6df880c6a40_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ba67a62d152342d8a994e6df880c6a40_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,277 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c2cbf72c04c447e58fcb929b36b5cc94_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c2cbf72c04c447e58fcb929b36b5cc94_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,279 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120df92cccc981b4c4badb0d10c124232fe_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120df92cccc981b4c4badb0d10c124232fe_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,280 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e2efce93a0b74839b74bf713ec2dbcac_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e2efce93a0b74839b74bf713ec2dbcac_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,282 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e63ce0ced1174997a822d2dab3bc2b6c_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e63ce0ced1174997a822d2dab3bc2b6c_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,283 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f89b2f027d214772a6633f2df6b91422_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f89b2f027d214772a6633f2df6b91422_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,284 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f89d289943df473fabc95bd0482d6c3e_5cb6af7207e610b3daed9e5466ca8411 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f89d289943df473fabc95bd0482d6c3e_5cb6af7207e610b3daed9e5466ca8411 2024-11-20T16:54:14,284 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T16:54:14,287 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=127, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:54:14,289 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T16:54:14,291 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T16:54:14,292 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=127, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:54:14,292 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T16:54:14,292 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732121654292"}]},"ts":"9223372036854775807"} 2024-11-20T16:54:14,293 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T16:54:14,294 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5cb6af7207e610b3daed9e5466ca8411, NAME => 'TestAcidGuarantees,,1732121617962.5cb6af7207e610b3daed9e5466ca8411.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T16:54:14,294 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T16:54:14,294 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732121654294"}]},"ts":"9223372036854775807"} 2024-11-20T16:54:14,295 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T16:54:14,296 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=127, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:54:14,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 79 msec 2024-11-20T16:54:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T16:54:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T16:54:14,330 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:14,340 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 250), OpenFileDescriptor=441 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=603 (was 729), ProcessCount=11 (was 11), AvailableMemoryMB=7490 (was 5644) - AvailableMemoryMB LEAK? - 2024-11-20T16:54:14,349 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=603, ProcessCount=11, AvailableMemoryMB=7489 2024-11-20T16:54:14,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T16:54:14,351 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T16:54:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T16:54:14,353 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T16:54:14,353 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:14,353 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 128 2024-11-20T16:54:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-20T16:54:14,353 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T16:54:14,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742309_1485 (size=1061) 2024-11-20T16:54:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-20T16:54:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-20T16:54:14,762 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625 2024-11-20T16:54:14,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742310_1486 (size=53) 2024-11-20T16:54:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-20T16:54:15,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:54:15,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 81890a01db534dee41b8dda5131aae4e, disabling compactions & flushes 2024-11-20T16:54:15,172 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:15,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:15,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. after waiting 0 ms 2024-11-20T16:54:15,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:15,172 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:15,173 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 81890a01db534dee41b8dda5131aae4e: Waiting for close lock at 1732121655172Disabling compacts and flushes for region at 1732121655172Disabling writes for close at 1732121655172Writing region close event to WAL at 1732121655172Closed at 1732121655172 2024-11-20T16:54:15,175 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T16:54:15,175 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732121655175"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732121655175"}]},"ts":"1732121655175"} 2024-11-20T16:54:15,178 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T16:54:15,180 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T16:54:15,180 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121655180"}]},"ts":"1732121655180"} 2024-11-20T16:54:15,183 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T16:54:15,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81890a01db534dee41b8dda5131aae4e, ASSIGN}] 2024-11-20T16:54:15,185 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81890a01db534dee41b8dda5131aae4e, ASSIGN 2024-11-20T16:54:15,186 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81890a01db534dee41b8dda5131aae4e, ASSIGN; state=OFFLINE, location=b60eac9bda43,33929,1732121526384; forceNewPlan=false, retain=false 2024-11-20T16:54:15,337 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=81890a01db534dee41b8dda5131aae4e, regionState=OPENING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:15,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81890a01db534dee41b8dda5131aae4e, ASSIGN because future has completed 2024-11-20T16:54:15,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:54:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-20T16:54:15,493 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:15,493 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7752): Opening region: {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} 2024-11-20T16:54:15,493 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:15,493 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:54:15,493 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7794): checking encryption for 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:15,493 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7797): checking classloading for 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:15,495 INFO [StoreOpener-81890a01db534dee41b8dda5131aae4e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:15,496 INFO [StoreOpener-81890a01db534dee41b8dda5131aae4e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:54:15,496 INFO [StoreOpener-81890a01db534dee41b8dda5131aae4e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81890a01db534dee41b8dda5131aae4e columnFamilyName A 2024-11-20T16:54:15,496 DEBUG [StoreOpener-81890a01db534dee41b8dda5131aae4e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:15,496 INFO [StoreOpener-81890a01db534dee41b8dda5131aae4e-1 {}] regionserver.HStore(327): Store=81890a01db534dee41b8dda5131aae4e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:54:15,496 INFO [StoreOpener-81890a01db534dee41b8dda5131aae4e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:15,497 INFO [StoreOpener-81890a01db534dee41b8dda5131aae4e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:54:15,497 INFO [StoreOpener-81890a01db534dee41b8dda5131aae4e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81890a01db534dee41b8dda5131aae4e columnFamilyName B 2024-11-20T16:54:15,497 DEBUG [StoreOpener-81890a01db534dee41b8dda5131aae4e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:15,498 INFO [StoreOpener-81890a01db534dee41b8dda5131aae4e-1 {}] regionserver.HStore(327): Store=81890a01db534dee41b8dda5131aae4e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:54:15,498 INFO [StoreOpener-81890a01db534dee41b8dda5131aae4e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:15,498 INFO [StoreOpener-81890a01db534dee41b8dda5131aae4e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:54:15,498 INFO [StoreOpener-81890a01db534dee41b8dda5131aae4e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81890a01db534dee41b8dda5131aae4e columnFamilyName C 2024-11-20T16:54:15,499 DEBUG [StoreOpener-81890a01db534dee41b8dda5131aae4e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:15,499 INFO [StoreOpener-81890a01db534dee41b8dda5131aae4e-1 {}] regionserver.HStore(327): Store=81890a01db534dee41b8dda5131aae4e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:54:15,499 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:15,499 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1038): replaying wal for 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:15,500 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:15,500 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:15,501 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1048): stopping wal replay for 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:15,501 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1060): Cleaning up temporary data for 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:15,501 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T16:54:15,502 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1093): writing seq id for 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:15,503 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T16:54:15,503 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1114): Opened 81890a01db534dee41b8dda5131aae4e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61679459, jitterRate=-0.08090443909168243}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T16:54:15,503 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:15,504 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1006): Region open journal for 81890a01db534dee41b8dda5131aae4e: Running coprocessor pre-open hook at 1732121655493Writing region info on filesystem at 1732121655493Initializing all the Stores at 1732121655494 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121655494Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121655494Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121655494Cleaning up temporary data from old regions at 1732121655501 (+7 ms)Running coprocessor post-open hooks at 1732121655503 (+2 ms)Region opened successfully at 1732121655504 (+1 ms) 2024-11-20T16:54:15,505 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., pid=130, masterSystemTime=1732121655490 2024-11-20T16:54:15,506 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:15,506 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:15,506 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=81890a01db534dee41b8dda5131aae4e, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:15,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 because future has completed 2024-11-20T16:54:15,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-20T16:54:15,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; OpenRegionProcedure 81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 in 169 msec 2024-11-20T16:54:15,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-20T16:54:15,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81890a01db534dee41b8dda5131aae4e, ASSIGN in 327 msec 2024-11-20T16:54:15,512 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T16:54:15,512 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121655512"}]},"ts":"1732121655512"} 2024-11-20T16:54:15,513 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T16:54:15,514 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T16:54:15,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-20T16:54:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-20T16:54:16,496 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:16,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a364c5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,497 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:16,498 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:16,498 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:16,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:16,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:16,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302e6f1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:16,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:16,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:16,500 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:16,500 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1830001a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,501 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:16,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:16,504 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T16:54:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is b60eac9bda43,33377,1732121525625 2024-11-20T16:54:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@55e42e08 2024-11-20T16:54:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T16:54:16,505 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T16:54:16,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6516064e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,507 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:16,507 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:16,507 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:16,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:16,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54be19b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:16,508 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39593382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fe0696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,510 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:16,510 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:16,510 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@232dbc0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:16,511 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75abcc3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,512 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:16,512 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:16,512 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74d61994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:16,514 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:16,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a951095, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,515 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:16,515 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:16,516 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c9040a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:16,517 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@105395ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,518 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:16,518 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:16,519 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:16,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:16,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:16,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54722273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:16,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:16,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:16,521 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:16,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57a764ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,522 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:16,522 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:16,523 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b3cdb5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:16,524 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77147106, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,525 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:16,525 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:16,526 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:16,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:16,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:16,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d9c874d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:16,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:16,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:16,527 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@724c3924, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,528 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:16,528 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:16,528 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:16,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14277c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:16,529 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:16,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c55fcad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,531 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:16,531 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:16,531 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af2dbc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:16,533 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:16,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd28502, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,534 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:16,534 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:16,535 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e759661, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:16,536 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@321a6d13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:16,537 DEBUG [Thread-2162 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:16,537 DEBUG [Thread-2160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:16,537 DEBUG [Thread-2156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:16,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:16,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:16,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:16,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:16,538 DEBUG [Thread-2169 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:16,538 DEBUG [Thread-2175 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:16,539 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:16,539 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:16,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:16,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:16,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:16,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:16,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:16,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:16,540 DEBUG [Thread-2158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:16,540 DEBUG [Thread-2164 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:54:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:16,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:54:16,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:16,542 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:16,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:16,542 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:54:16,543 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:54:16,544 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:54:16,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:54:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:54:16,547 DEBUG [Thread-2171 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:16,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:16,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:16,548 DEBUG [Thread-2167 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:16,549 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:54:16,549 DEBUG [Thread-2173 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:16,550 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:16,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:16,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:16,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:16,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:16,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:54:16,551 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:16,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-20T16:54:16,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:54:16,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2] 2024-11-20T16:54:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T16:54:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:16,555 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:54:16,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T16:54:16,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:16,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:16,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:16,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:16,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:16,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:16,556 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:54:16,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:54:16,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/ae697bbced1f4ae6aadf534eaf2e1773 is 50, key is test_row_0/A:col10/1732121656551/Put/seqid=0 2024-11-20T16:54:16,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742311_1487 (size=12001) 2024-11-20T16:54:16,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/ae697bbced1f4ae6aadf534eaf2e1773 2024-11-20T16:54:16,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121716589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:16,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732121656584, max timestamp=1732121656602Num uniques 859; 2024-11-20T16:54:16,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732121656584, max timestamp=1732121656602Num uniques 859; 2024-11-20T16:54:16,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732121656584, max timestamp=1732121656602Num uniques 859; 2024-11-20T16:54:16,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T16:54:16,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T16:54:16,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T16:54:16,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/20908969045c4ca485eac43f6efa29cb is 50, key is test_row_0/B:col10/1732121656551/Put/seqid=0 2024-11-20T16:54:16,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742312_1488 (size=12001) 2024-11-20T16:54:16,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121716637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:16,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:16,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:16,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121716637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:16,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:16,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47156 deadline: 1732121716647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:16,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:16,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T16:54:16,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121716650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:16,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:16,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T16:54:16,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:16,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:16,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:16,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:16,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:16,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:16,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121716716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:16,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:16,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121716756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:16,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:16,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121716756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:16,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47156 deadline: 1732121716756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:16,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:16,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:16,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:16,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121716766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:16,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:16,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T16:54:16,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:16,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:16,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:16,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:16,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:16,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T16:54:16,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121716936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:16,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:16,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121716976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:16,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47156 deadline: 1732121716976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:16,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:16,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121716976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:16,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:16,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121716986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:17,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T16:54:17,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:17,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:17,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:17,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:17,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:17,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/20908969045c4ca485eac43f6efa29cb 2024-11-20T16:54:17,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/3e2c4f3d5a104eb494d8c8ff6ca3e2cd is 50, key is test_row_0/C:col10/1732121656551/Put/seqid=0 2024-11-20T16:54:17,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742313_1489 (size=12001) 2024-11-20T16:54:17,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T16:54:17,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:17,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:17,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:17,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:17,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T16:54:17,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121717246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:17,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:17,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121717296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:17,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47156 deadline: 1732121717296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:17,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121717296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:17,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:17,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:17,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121717297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:17,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:17,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T16:54:17,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:17,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:17,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:17,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:17,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:17,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:17,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/3e2c4f3d5a104eb494d8c8ff6ca3e2cd 2024-11-20T16:54:17,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/ae697bbced1f4ae6aadf534eaf2e1773 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/ae697bbced1f4ae6aadf534eaf2e1773 2024-11-20T16:54:17,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T16:54:17,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:17,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:17,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:17,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:17,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:17,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/ae697bbced1f4ae6aadf534eaf2e1773, entries=150, sequenceid=12, filesize=11.7 K 2024-11-20T16:54:17,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:17,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/20908969045c4ca485eac43f6efa29cb as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/20908969045c4ca485eac43f6efa29cb 2024-11-20T16:54:17,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/20908969045c4ca485eac43f6efa29cb, entries=150, sequenceid=12, filesize=11.7 K 2024-11-20T16:54:17,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/3e2c4f3d5a104eb494d8c8ff6ca3e2cd as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/3e2c4f3d5a104eb494d8c8ff6ca3e2cd 2024-11-20T16:54:17,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/3e2c4f3d5a104eb494d8c8ff6ca3e2cd, entries=150, sequenceid=12, filesize=11.7 K 2024-11-20T16:54:17,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 81890a01db534dee41b8dda5131aae4e in 935ms, sequenceid=12, compaction requested=false 2024-11-20T16:54:17,490 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T16:54:17,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:17,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T16:54:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:17,629 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-20T16:54:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:17,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:17,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:17,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/5d4722ff2f9049cbb2f77038f8eb9d9f is 50, key is test_row_0/A:col10/1732121656648/Put/seqid=0 2024-11-20T16:54:17,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742314_1490 (size=12001) 2024-11-20T16:54:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T16:54:17,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:17,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:17,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121717816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:17,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47156 deadline: 1732121717816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121717816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121717816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:17,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121717822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:17,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121717936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:18,037 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/5d4722ff2f9049cbb2f77038f8eb9d9f 2024-11-20T16:54:18,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/3b4a2358a6c54ec9abf37088a42a1b13 is 50, key is test_row_0/B:col10/1732121656648/Put/seqid=0 2024-11-20T16:54:18,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742315_1491 (size=12001) 2024-11-20T16:54:18,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121718157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:18,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:18,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:18,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:18,452 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/3b4a2358a6c54ec9abf37088a42a1b13 2024-11-20T16:54:18,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/14ed59d87ab34909bc64f194d97264dd is 50, key is test_row_0/C:col10/1732121656648/Put/seqid=0 2024-11-20T16:54:18,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742316_1492 (size=12001) 2024-11-20T16:54:18,469 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/14ed59d87ab34909bc64f194d97264dd 2024-11-20T16:54:18,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/5d4722ff2f9049cbb2f77038f8eb9d9f as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/5d4722ff2f9049cbb2f77038f8eb9d9f 2024-11-20T16:54:18,484 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/5d4722ff2f9049cbb2f77038f8eb9d9f, entries=150, sequenceid=46, filesize=11.7 K 2024-11-20T16:54:18,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121718476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:18,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/3b4a2358a6c54ec9abf37088a42a1b13 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/3b4a2358a6c54ec9abf37088a42a1b13 2024-11-20T16:54:18,489 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/3b4a2358a6c54ec9abf37088a42a1b13, entries=150, sequenceid=46, filesize=11.7 K 2024-11-20T16:54:18,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/14ed59d87ab34909bc64f194d97264dd as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/14ed59d87ab34909bc64f194d97264dd 2024-11-20T16:54:18,494 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/14ed59d87ab34909bc64f194d97264dd, entries=150, sequenceid=46, filesize=11.7 K 2024-11-20T16:54:18,495 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 81890a01db534dee41b8dda5131aae4e in 866ms, sequenceid=46, compaction requested=false 2024-11-20T16:54:18,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:18,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:18,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-20T16:54:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-20T16:54:18,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-20T16:54:18,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9410 sec 2024-11-20T16:54:18,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.9500 sec 2024-11-20T16:54:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T16:54:18,696 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:18,698 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:54:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-20T16:54:18,700 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:54:18,702 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:54:18,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:54:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T16:54:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T16:54:18,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T16:54:18,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:18,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:18,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:18,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:18,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:18,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:18,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T16:54:18,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:18,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:18,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:18,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:18,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:18,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/ca8d834242ce42c48156df3dec5ae51d is 50, key is test_row_0/A:col10/1732121658845/Put/seqid=0 2024-11-20T16:54:18,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742317_1493 (size=16681) 2024-11-20T16:54:18,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121718910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:18,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:18,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:18,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:18,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47156 deadline: 1732121718911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:18,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:18,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:18,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:18,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121718923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:18,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121718925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:18,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:18,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:18,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:19,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121718996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:19,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:19,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T16:54:19,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:19,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:19,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:19,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T16:54:19,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47156 deadline: 1732121719036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:19,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121719036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121719037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:19,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121719046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:19,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:19,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T16:54:19,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:19,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:19,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:19,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47156 deadline: 1732121719256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:19,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121719256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:19,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121719256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:19,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121719256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:19,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:19,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/ca8d834242ce42c48156df3dec5ae51d 2024-11-20T16:54:19,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/7a70dd2c3e3a4cc4a66d723cfd6290c5 is 50, key is test_row_0/B:col10/1732121658845/Put/seqid=0 2024-11-20T16:54:19,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742318_1494 (size=12001) 2024-11-20T16:54:19,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T16:54:19,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:19,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:19,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:19,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T16:54:19,415 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T16:54:19,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T16:54:19,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:19,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:19,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:19,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121719566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:19,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121719567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:19,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47156 deadline: 1732121719567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:19,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121719576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:19,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:19,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:19,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T16:54:19,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:19,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:19,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:19,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/7a70dd2c3e3a4cc4a66d723cfd6290c5 2024-11-20T16:54:19,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/ee7b2222c19148468c4e4acc76e8bcea is 50, key is test_row_0/C:col10/1732121658845/Put/seqid=0 2024-11-20T16:54:19,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742319_1495 (size=12001) 2024-11-20T16:54:19,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T16:54:19,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:19,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:19,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:19,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T16:54:19,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T16:54:19,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:19,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:19,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:19,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:20,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121720016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:20,025 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 81890a01db534dee41b8dda5131aae4e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:20,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:20,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:20,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:20,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T16:54:20,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:20,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:20,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:20,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:20,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121720076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:20,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47156 deadline: 1732121720076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:20,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:20,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:20,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:20,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:20,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:20,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:20,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121720076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:20,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:20,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:20,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:20,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121720096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:20,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/ee7b2222c19148468c4e4acc76e8bcea 2024-11-20T16:54:20,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/ca8d834242ce42c48156df3dec5ae51d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/ca8d834242ce42c48156df3dec5ae51d 2024-11-20T16:54:20,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/ca8d834242ce42c48156df3dec5ae51d, entries=250, sequenceid=59, filesize=16.3 K 2024-11-20T16:54:20,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/7a70dd2c3e3a4cc4a66d723cfd6290c5 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/7a70dd2c3e3a4cc4a66d723cfd6290c5 2024-11-20T16:54:20,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/7a70dd2c3e3a4cc4a66d723cfd6290c5, entries=150, sequenceid=59, filesize=11.7 K 2024-11-20T16:54:20,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/ee7b2222c19148468c4e4acc76e8bcea as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/ee7b2222c19148468c4e4acc76e8bcea 2024-11-20T16:54:20,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/ee7b2222c19148468c4e4acc76e8bcea, entries=150, sequenceid=59, filesize=11.7 K 2024-11-20T16:54:20,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 81890a01db534dee41b8dda5131aae4e in 1292ms, sequenceid=59, compaction requested=true 2024-11-20T16:54:20,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:20,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:54:20,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:20,139 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:54:20,139 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:54:20,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:54:20,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:20,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:54:20,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:20,140 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:54:20,140 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:54:20,140 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/A is initiating minor compaction (all files) 2024-11-20T16:54:20,140 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/B is initiating minor compaction (all files) 2024-11-20T16:54:20,140 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/B in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:20,140 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/20908969045c4ca485eac43f6efa29cb, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/3b4a2358a6c54ec9abf37088a42a1b13, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/7a70dd2c3e3a4cc4a66d723cfd6290c5] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=35.2 K 2024-11-20T16:54:20,140 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/A in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:20,141 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/ae697bbced1f4ae6aadf534eaf2e1773, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/5d4722ff2f9049cbb2f77038f8eb9d9f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/ca8d834242ce42c48156df3dec5ae51d] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=39.7 K 2024-11-20T16:54:20,141 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 20908969045c4ca485eac43f6efa29cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732121656547 2024-11-20T16:54:20,141 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae697bbced1f4ae6aadf534eaf2e1773, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732121656547 2024-11-20T16:54:20,141 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d4722ff2f9049cbb2f77038f8eb9d9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732121656636 2024-11-20T16:54:20,141 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca8d834242ce42c48156df3dec5ae51d, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732121657796 2024-11-20T16:54:20,141 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b4a2358a6c54ec9abf37088a42a1b13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732121656636 2024-11-20T16:54:20,142 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a70dd2c3e3a4cc4a66d723cfd6290c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732121657820 2024-11-20T16:54:20,157 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#B#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:20,157 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#A#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:20,157 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/6ce72fdf97844993be4321e03835365d is 50, key is test_row_0/B:col10/1732121658845/Put/seqid=0 2024-11-20T16:54:20,157 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/cdfc27b7975249aebc1d0df8033647d8 is 50, key is test_row_0/A:col10/1732121658845/Put/seqid=0 2024-11-20T16:54:20,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742320_1496 (size=12104) 2024-11-20T16:54:20,174 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/cdfc27b7975249aebc1d0df8033647d8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/cdfc27b7975249aebc1d0df8033647d8 2024-11-20T16:54:20,179 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/A of 81890a01db534dee41b8dda5131aae4e into cdfc27b7975249aebc1d0df8033647d8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:20,179 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:20,179 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/A, priority=13, startTime=1732121660139; duration=0sec 2024-11-20T16:54:20,180 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:20,180 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:A 2024-11-20T16:54:20,180 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:54:20,180 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:54:20,180 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/C is initiating minor compaction (all files) 2024-11-20T16:54:20,181 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/C in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:20,181 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/3e2c4f3d5a104eb494d8c8ff6ca3e2cd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/14ed59d87ab34909bc64f194d97264dd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/ee7b2222c19148468c4e4acc76e8bcea] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=35.2 K 2024-11-20T16:54:20,181 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e2c4f3d5a104eb494d8c8ff6ca3e2cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732121656547 2024-11-20T16:54:20,181 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14ed59d87ab34909bc64f194d97264dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732121656636 2024-11-20T16:54:20,181 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee7b2222c19148468c4e4acc76e8bcea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732121657820 2024-11-20T16:54:20,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742321_1497 (size=12104) 2024-11-20T16:54:20,187 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/6ce72fdf97844993be4321e03835365d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/6ce72fdf97844993be4321e03835365d 2024-11-20T16:54:20,188 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#C#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:20,188 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/d4c7a59ee0d143c992e56163dfab3de5 is 50, key is test_row_0/C:col10/1732121658845/Put/seqid=0 2024-11-20T16:54:20,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742322_1498 (size=12104) 2024-11-20T16:54:20,194 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/B of 81890a01db534dee41b8dda5131aae4e into 6ce72fdf97844993be4321e03835365d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:20,194 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:20,194 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/B, priority=13, startTime=1732121660139; duration=0sec 2024-11-20T16:54:20,194 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:20,194 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:B 2024-11-20T16:54:20,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T16:54:20,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:20,234 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T16:54:20,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:20,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:20,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:20,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:20,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:20,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:20,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/907712d1d05447d1bd432637d3bd6b4d is 50, key is test_row_0/A:col10/1732121658909/Put/seqid=0 2024-11-20T16:54:20,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742323_1499 (size=12001) 2024-11-20T16:54:20,247 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/907712d1d05447d1bd432637d3bd6b4d 2024-11-20T16:54:20,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/0fb829d766ee47929da90087d2937590 is 50, key is test_row_0/B:col10/1732121658909/Put/seqid=0 2024-11-20T16:54:20,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742324_1500 (size=12001) 2024-11-20T16:54:20,596 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/d4c7a59ee0d143c992e56163dfab3de5 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/d4c7a59ee0d143c992e56163dfab3de5 2024-11-20T16:54:20,599 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/C of 81890a01db534dee41b8dda5131aae4e into d4c7a59ee0d143c992e56163dfab3de5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:20,599 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:20,599 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/C, priority=13, startTime=1732121660139; duration=0sec 2024-11-20T16:54:20,599 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:20,599 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:C 2024-11-20T16:54:20,659 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/0fb829d766ee47929da90087d2937590 2024-11-20T16:54:20,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/25f6dc0288354621bcdc8092510dccaf is 50, key is test_row_0/C:col10/1732121658909/Put/seqid=0 2024-11-20T16:54:20,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742325_1501 (size=12001) 2024-11-20T16:54:20,671 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/25f6dc0288354621bcdc8092510dccaf 2024-11-20T16:54:20,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/907712d1d05447d1bd432637d3bd6b4d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/907712d1d05447d1bd432637d3bd6b4d 2024-11-20T16:54:20,678 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/907712d1d05447d1bd432637d3bd6b4d, entries=150, sequenceid=80, filesize=11.7 K 2024-11-20T16:54:20,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/0fb829d766ee47929da90087d2937590 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/0fb829d766ee47929da90087d2937590 2024-11-20T16:54:20,681 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/0fb829d766ee47929da90087d2937590, entries=150, sequenceid=80, filesize=11.7 K 2024-11-20T16:54:20,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-20T16:54:20,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/25f6dc0288354621bcdc8092510dccaf as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/25f6dc0288354621bcdc8092510dccaf 2024-11-20T16:54:20,685 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/25f6dc0288354621bcdc8092510dccaf, entries=150, sequenceid=80, filesize=11.7 K 2024-11-20T16:54:20,686 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 81890a01db534dee41b8dda5131aae4e in 452ms, sequenceid=80, compaction requested=false 2024-11-20T16:54:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-20T16:54:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-20T16:54:20,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-20T16:54:20,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9860 sec 2024-11-20T16:54:20,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.9920 sec 2024-11-20T16:54:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T16:54:20,846 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:20,847 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:54:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-20T16:54:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T16:54:20,849 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:54:20,851 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:54:20,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:54:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T16:54:21,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T16:54:21,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-20T16:54:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-20T16:54:21,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-20T16:54:21,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-20T16:54:21,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 162 msec 2024-11-20T16:54:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:21,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T16:54:21,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:21,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:21,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:21,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:21,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:21,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:21,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/ccd137b969294ba2bccc6db21ea7de7e is 50, key is test_row_0/A:col10/1732121661100/Put/seqid=0 2024-11-20T16:54:21,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742326_1502 (size=9657) 2024-11-20T16:54:21,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121661140, max timestamp=1732121661147Num uniques 859; 2024-11-20T16:54:21,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121661140, max timestamp=1732121661147Num uniques 859; 2024-11-20T16:54:21,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:21,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:21,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121661140, max timestamp=1732121661147Num uniques 859; 2024-11-20T16:54:21,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:21,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T16:54:21,166 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:21,167 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:54:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-20T16:54:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T16:54:21,168 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:54:21,168 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:54:21,168 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:54:21,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:21,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121721189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:21,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:21,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47156 deadline: 1732121721190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:21,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121721191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:21,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121721192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:21,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T16:54:21,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121721296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:21,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:21,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47156 deadline: 1732121721306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:21,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T16:54:21,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:21,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:21,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:21,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:21,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:21,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:21,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:21,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121721316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:21,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:21,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121721316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:21,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:21,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:21,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T16:54:21,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:21,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:21,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:21,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:21,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T16:54:21,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/ccd137b969294ba2bccc6db21ea7de7e 2024-11-20T16:54:21,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:21,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121721506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:21,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:21,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/68565cd2e75a43bc97db025caef07725 is 50, key is test_row_0/B:col10/1732121661100/Put/seqid=0 2024-11-20T16:54:21,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742327_1503 (size=9657) 2024-11-20T16:54:21,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47156 deadline: 1732121721516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:21,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:21,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121721536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:21,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:21,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121721536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:21,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:21,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:21,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T16:54:21,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:21,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:21,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:21,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:21,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:21,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:21,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T16:54:21,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:21,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:21,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:21,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:21,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T16:54:21,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121721826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:21,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47156 deadline: 1732121721836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121721846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:21,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121721856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:21,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:21,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:21,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/68565cd2e75a43bc97db025caef07725 2024-11-20T16:54:21,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T16:54:21,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:21,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:21,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:21,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:21,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:21,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/849953e9b55245819fc2f0aef206ea0d is 50, key is test_row_0/C:col10/1732121661100/Put/seqid=0 2024-11-20T16:54:21,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742328_1504 (size=9657) 2024-11-20T16:54:22,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121722046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:22,058 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 81890a01db534dee41b8dda5131aae4e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:22,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T16:54:22,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:22,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:22,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:22,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:22,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:22,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T16:54:22,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:22,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:22,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:22,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:22,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T16:54:22,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121722346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:22,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:22,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:22,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:22,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/849953e9b55245819fc2f0aef206ea0d 2024-11-20T16:54:22,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/ccd137b969294ba2bccc6db21ea7de7e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/ccd137b969294ba2bccc6db21ea7de7e 2024-11-20T16:54:22,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/ccd137b969294ba2bccc6db21ea7de7e, entries=100, sequenceid=93, filesize=9.4 K 2024-11-20T16:54:22,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/68565cd2e75a43bc97db025caef07725 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/68565cd2e75a43bc97db025caef07725 2024-11-20T16:54:22,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/68565cd2e75a43bc97db025caef07725, entries=100, sequenceid=93, filesize=9.4 K 2024-11-20T16:54:22,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/849953e9b55245819fc2f0aef206ea0d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/849953e9b55245819fc2f0aef206ea0d 2024-11-20T16:54:22,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47156 deadline: 1732121722366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:22,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:22,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:22,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:22,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/849953e9b55245819fc2f0aef206ea0d, entries=100, sequenceid=93, filesize=9.4 K 2024-11-20T16:54:22,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 81890a01db534dee41b8dda5131aae4e in 1279ms, sequenceid=93, compaction requested=true 2024-11-20T16:54:22,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:22,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:54:22,380 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:54:22,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:22,380 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:54:22,381 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:54:22,381 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/A is initiating minor compaction (all files) 2024-11-20T16:54:22,381 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/A in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:22,381 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/cdfc27b7975249aebc1d0df8033647d8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/907712d1d05447d1bd432637d3bd6b4d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/ccd137b969294ba2bccc6db21ea7de7e] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=33.0 K 2024-11-20T16:54:22,382 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdfc27b7975249aebc1d0df8033647d8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732121657820 2024-11-20T16:54:22,382 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 907712d1d05447d1bd432637d3bd6b4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732121658882 2024-11-20T16:54:22,382 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccd137b969294ba2bccc6db21ea7de7e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732121661100 2024-11-20T16:54:22,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:54:22,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:22,384 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:54:22,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:54:22,384 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/B is initiating minor compaction (all files) 2024-11-20T16:54:22,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:22,384 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/B in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:22,384 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/6ce72fdf97844993be4321e03835365d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/0fb829d766ee47929da90087d2937590, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/68565cd2e75a43bc97db025caef07725] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=33.0 K 2024-11-20T16:54:22,385 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ce72fdf97844993be4321e03835365d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732121657820 2024-11-20T16:54:22,385 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fb829d766ee47929da90087d2937590, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732121658882 2024-11-20T16:54:22,385 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 68565cd2e75a43bc97db025caef07725, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732121661100 2024-11-20T16:54:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:22,388 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#A#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:22,388 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/75c613c5a41d4a5f9b60bba813383be3 is 50, key is test_row_0/A:col10/1732121661100/Put/seqid=0 2024-11-20T16:54:22,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-20T16:54:22,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:22,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:22,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:22,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:22,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:22,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:22,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T16:54:22,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:22,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:22,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:22,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:22,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/bd5d619808264156945ff1134beb5930 is 50, key is test_row_0/A:col10/1732121661173/Put/seqid=0 2024-11-20T16:54:22,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:22,428 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#B#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:22,429 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/715fb96de16f463aa2e931716e95744a is 50, key is test_row_0/B:col10/1732121661100/Put/seqid=0 2024-11-20T16:54:22,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742329_1505 (size=12207) 2024-11-20T16:54:22,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121722443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:22,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:22,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:22,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:22,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121722443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:22,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742330_1506 (size=16781) 2024-11-20T16:54:22,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/bd5d619808264156945ff1134beb5930 2024-11-20T16:54:22,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742331_1507 (size=12207) 2024-11-20T16:54:22,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/4f2e6f73581a4250b0b390ef36f8dfc3 is 50, key is test_row_0/B:col10/1732121661173/Put/seqid=0 2024-11-20T16:54:22,466 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/715fb96de16f463aa2e931716e95744a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/715fb96de16f463aa2e931716e95744a 2024-11-20T16:54:22,473 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/B of 81890a01db534dee41b8dda5131aae4e into 715fb96de16f463aa2e931716e95744a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:22,473 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:22,473 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/B, priority=13, startTime=1732121662380; duration=0sec 2024-11-20T16:54:22,473 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:22,473 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:B 2024-11-20T16:54:22,473 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:54:22,474 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:54:22,474 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/C is initiating minor compaction (all files) 2024-11-20T16:54:22,474 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/C in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:22,474 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/d4c7a59ee0d143c992e56163dfab3de5, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/25f6dc0288354621bcdc8092510dccaf, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/849953e9b55245819fc2f0aef206ea0d] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=33.0 K 2024-11-20T16:54:22,475 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting d4c7a59ee0d143c992e56163dfab3de5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732121657820 2024-11-20T16:54:22,475 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 25f6dc0288354621bcdc8092510dccaf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732121658882 2024-11-20T16:54:22,475 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 849953e9b55245819fc2f0aef206ea0d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732121661100 2024-11-20T16:54:22,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742332_1508 (size=12051) 2024-11-20T16:54:22,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/4f2e6f73581a4250b0b390ef36f8dfc3 2024-11-20T16:54:22,494 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#C#compaction#422 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:22,495 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/5ae5757d30314a519fc4f0610239dd69 is 50, key is test_row_0/C:col10/1732121661100/Put/seqid=0 2024-11-20T16:54:22,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742333_1509 (size=12207) 2024-11-20T16:54:22,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/6864a74719a04d45a819054e9341e4c4 is 50, key is test_row_0/C:col10/1732121661173/Put/seqid=0 2024-11-20T16:54:22,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T16:54:22,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:22,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:22,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:22,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:22,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:22,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121722546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:22,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742334_1510 (size=12051) 2024-11-20T16:54:22,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/6864a74719a04d45a819054e9341e4c4 2024-11-20T16:54:22,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/bd5d619808264156945ff1134beb5930 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/bd5d619808264156945ff1134beb5930 2024-11-20T16:54:22,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/bd5d619808264156945ff1134beb5930, entries=250, sequenceid=130, filesize=16.4 K 2024-11-20T16:54:22,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121722558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:22,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/4f2e6f73581a4250b0b390ef36f8dfc3 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/4f2e6f73581a4250b0b390ef36f8dfc3 2024-11-20T16:54:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:22,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/4f2e6f73581a4250b0b390ef36f8dfc3, entries=150, sequenceid=130, filesize=11.8 K 2024-11-20T16:54:22,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/6864a74719a04d45a819054e9341e4c4 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/6864a74719a04d45a819054e9341e4c4 2024-11-20T16:54:22,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/6864a74719a04d45a819054e9341e4c4, entries=150, sequenceid=130, filesize=11.8 K 2024-11-20T16:54:22,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 81890a01db534dee41b8dda5131aae4e in 199ms, sequenceid=130, compaction requested=false 2024-11-20T16:54:22,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:22,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T16:54:22,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:22,704 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-20T16:54:22,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:22,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:22,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:22,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:22,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:22,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:22,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/a94f72503db64c56a2cf0dee86901ddf is 50, key is test_row_1/A:col10/1732121662441/Put/seqid=0 2024-11-20T16:54:22,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742335_1511 (size=9757) 2024-11-20T16:54:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:22,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:22,848 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/75c613c5a41d4a5f9b60bba813383be3 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/75c613c5a41d4a5f9b60bba813383be3 2024-11-20T16:54:22,854 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/A of 81890a01db534dee41b8dda5131aae4e into 75c613c5a41d4a5f9b60bba813383be3(size=11.9 K), total size for store is 28.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:22,854 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:22,854 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/A, priority=13, startTime=1732121662380; duration=0sec 2024-11-20T16:54:22,854 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:22,854 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:A 2024-11-20T16:54:22,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121662839, max timestamp=1732121662900Num uniques 859; 2024-11-20T16:54:22,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121662839, max timestamp=1732121662900Num uniques 859; 2024-11-20T16:54:22,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:54:22,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:54:22,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121662839, max timestamp=1732121662900Num uniques 859; 2024-11-20T16:54:22,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:54:22,947 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/5ae5757d30314a519fc4f0610239dd69 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/5ae5757d30314a519fc4f0610239dd69 2024-11-20T16:54:22,954 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/C of 81890a01db534dee41b8dda5131aae4e into 5ae5757d30314a519fc4f0610239dd69(size=11.9 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:22,954 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:22,954 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/C, priority=13, startTime=1732121662384; duration=0sec 2024-11-20T16:54:22,954 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:22,954 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:C 2024-11-20T16:54:23,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732121662979, max timestamp=1732121662997Num uniques 1018; 2024-11-20T16:54:23,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T16:54:23,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732121662979, max timestamp=1732121662997Num uniques 1018; 2024-11-20T16:54:23,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T16:54:23,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732121662979, max timestamp=1732121662997Num uniques 1018; 2024-11-20T16:54:23,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T16:54:23,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732121663039, max timestamp=1732121663113Num uniques 1027; 2024-11-20T16:54:23,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-20T16:54:23,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732121663039, max timestamp=1732121663113Num uniques 1027; 2024-11-20T16:54:23,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-20T16:54:23,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732121663039, max timestamp=1732121663113Num uniques 1027; 2024-11-20T16:54:23,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-20T16:54:23,121 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/a94f72503db64c56a2cf0dee86901ddf 2024-11-20T16:54:23,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/6b741fce5f2e4480843fd079bf1b8de6 is 50, key is test_row_1/B:col10/1732121662441/Put/seqid=0 2024-11-20T16:54:23,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742336_1512 (size=9757) 2024-11-20T16:54:23,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732121663176, max timestamp=1732121663221Num uniques 1036; 2024-11-20T16:54:23,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-20T16:54:23,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732121663176, max timestamp=1732121663221Num uniques 1036; 2024-11-20T16:54:23,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732121663176, max timestamp=1732121663221Num uniques 1036; 2024-11-20T16:54:23,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-20T16:54:23,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-20T16:54:23,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121723219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T16:54:23,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121723368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121723380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121723381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:23,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47156 deadline: 1732121723386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:23,396 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_0' in 81890a01db534dee41b8dda5131aae4e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:23,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:23,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121723496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:23,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121723506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:23,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121723506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:23,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:23,533 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/6b741fce5f2e4480843fd079bf1b8de6 2024-11-20T16:54:23,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/17adca6e4b494dbeadeded6ea348bf31 is 50, key is test_row_1/C:col10/1732121662441/Put/seqid=0 2024-11-20T16:54:23,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742337_1513 (size=9757) 2024-11-20T16:54:23,570 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/17adca6e4b494dbeadeded6ea348bf31 2024-11-20T16:54:23,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/a94f72503db64c56a2cf0dee86901ddf as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/a94f72503db64c56a2cf0dee86901ddf 2024-11-20T16:54:23,584 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/a94f72503db64c56a2cf0dee86901ddf, entries=100, sequenceid=140, filesize=9.5 K 2024-11-20T16:54:23,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/6b741fce5f2e4480843fd079bf1b8de6 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/6b741fce5f2e4480843fd079bf1b8de6 2024-11-20T16:54:23,592 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/6b741fce5f2e4480843fd079bf1b8de6, entries=100, sequenceid=140, filesize=9.5 K 2024-11-20T16:54:23,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/17adca6e4b494dbeadeded6ea348bf31 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/17adca6e4b494dbeadeded6ea348bf31 2024-11-20T16:54:23,597 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/17adca6e4b494dbeadeded6ea348bf31, entries=100, sequenceid=140, filesize=9.5 K 2024-11-20T16:54:23,598 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 81890a01db534dee41b8dda5131aae4e in 894ms, sequenceid=140, compaction requested=true 2024-11-20T16:54:23,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:23,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:23,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-20T16:54:23,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-20T16:54:23,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-20T16:54:23,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4340 sec 2024-11-20T16:54:23,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.4380 sec 2024-11-20T16:54:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:23,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=147.60 KB heapSize=418.31 KB 2024-11-20T16:54:23,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:23,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T16:54:23,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:23,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T16:54:23,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:23,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T16:54:23,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/6efdd1b69084470a9e9c813024862606 is 50, key is test_row_0/A:col10/1732121663380/Put/seqid=0 2024-11-20T16:54:23,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742338_1514 (size=14541) 2024-11-20T16:54:23,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121723770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:23,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121723770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:23,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121723772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:23,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121723897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:23,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:23,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121723897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:23,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:23,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:23,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121723897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:23,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:23,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:24,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121724116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:24,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121724126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:24,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:24,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:24,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121724126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:24,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:24,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/6efdd1b69084470a9e9c813024862606 2024-11-20T16:54:24,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/80c8bf92ed314580a28488d92e6f8260 is 50, key is test_row_0/B:col10/1732121663380/Put/seqid=0 2024-11-20T16:54:24,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742339_1515 (size=12151) 2024-11-20T16:54:24,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/80c8bf92ed314580a28488d92e6f8260 2024-11-20T16:54:24,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/bd272765bf0d4ae58e17f71a5352e6b3 is 50, key is test_row_0/C:col10/1732121663380/Put/seqid=0 2024-11-20T16:54:24,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742340_1516 (size=12151) 2024-11-20T16:54:24,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121724436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:24,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:24,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:24,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:24,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121724446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:24,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121724446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:24,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:24,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:24,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:24,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/bd272765bf0d4ae58e17f71a5352e6b3 2024-11-20T16:54:24,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/6efdd1b69084470a9e9c813024862606 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/6efdd1b69084470a9e9c813024862606 2024-11-20T16:54:24,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/6efdd1b69084470a9e9c813024862606, entries=200, sequenceid=231, filesize=14.2 K 2024-11-20T16:54:24,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/80c8bf92ed314580a28488d92e6f8260 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/80c8bf92ed314580a28488d92e6f8260 2024-11-20T16:54:24,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/80c8bf92ed314580a28488d92e6f8260, entries=150, sequenceid=231, filesize=11.9 K 2024-11-20T16:54:24,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/bd272765bf0d4ae58e17f71a5352e6b3 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/bd272765bf0d4ae58e17f71a5352e6b3 2024-11-20T16:54:24,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/bd272765bf0d4ae58e17f71a5352e6b3, entries=150, sequenceid=231, filesize=11.9 K 2024-11-20T16:54:24,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~418.27 KB/428304, currentSize=33.54 KB/34350 for 81890a01db534dee41b8dda5131aae4e in 879ms, sequenceid=231, compaction requested=true 2024-11-20T16:54:24,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:24,610 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:54:24,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:54:24,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:24,610 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:54:24,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:54:24,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:24,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:54:24,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:24,611 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53286 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:54:24,611 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/A is initiating minor compaction (all files) 2024-11-20T16:54:24,611 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/A in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:24,611 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/75c613c5a41d4a5f9b60bba813383be3, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/bd5d619808264156945ff1134beb5930, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/a94f72503db64c56a2cf0dee86901ddf, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/6efdd1b69084470a9e9c813024862606] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=52.0 K 2024-11-20T16:54:24,617 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46166 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:54:24,617 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/B is initiating minor compaction (all files) 2024-11-20T16:54:24,617 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/B in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:24,618 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/715fb96de16f463aa2e931716e95744a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/4f2e6f73581a4250b0b390ef36f8dfc3, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/6b741fce5f2e4480843fd079bf1b8de6, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/80c8bf92ed314580a28488d92e6f8260] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=45.1 K 2024-11-20T16:54:24,618 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 715fb96de16f463aa2e931716e95744a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732121658923 2024-11-20T16:54:24,618 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f2e6f73581a4250b0b390ef36f8dfc3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732121661173 2024-11-20T16:54:24,618 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b741fce5f2e4480843fd079bf1b8de6, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732121662421 2024-11-20T16:54:24,618 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 80c8bf92ed314580a28488d92e6f8260, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732121663358 2024-11-20T16:54:24,619 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75c613c5a41d4a5f9b60bba813383be3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732121658923 2024-11-20T16:54:24,619 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd5d619808264156945ff1134beb5930, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732121661173 2024-11-20T16:54:24,620 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting a94f72503db64c56a2cf0dee86901ddf, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732121662421 2024-11-20T16:54:24,620 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6efdd1b69084470a9e9c813024862606, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732121663358 2024-11-20T16:54:24,632 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#B#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:24,632 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#A#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:24,632 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/1a2dc0af6da44336aaee7796f229988c is 50, key is test_row_0/B:col10/1732121663380/Put/seqid=0 2024-11-20T16:54:24,632 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/badf0a45358647aaa08b3b849acf4714 is 50, key is test_row_0/A:col10/1732121663380/Put/seqid=0 2024-11-20T16:54:24,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742342_1518 (size=12493) 2024-11-20T16:54:24,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742341_1517 (size=12493) 2024-11-20T16:54:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:24,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T16:54:24,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:24,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:24,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:24,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:24,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:24,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:24,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/2243e286cc1d444abfd9a620572181dc is 50, key is test_row_0/A:col10/1732121663770/Put/seqid=0 2024-11-20T16:54:24,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742343_1519 (size=14541) 2024-11-20T16:54:25,055 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/badf0a45358647aaa08b3b849acf4714 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/badf0a45358647aaa08b3b849acf4714 2024-11-20T16:54:25,059 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/1a2dc0af6da44336aaee7796f229988c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/1a2dc0af6da44336aaee7796f229988c 2024-11-20T16:54:25,059 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/A of 81890a01db534dee41b8dda5131aae4e into badf0a45358647aaa08b3b849acf4714(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:25,059 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:25,059 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/A, priority=12, startTime=1732121664609; duration=0sec 2024-11-20T16:54:25,059 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:25,059 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:A 2024-11-20T16:54:25,059 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:54:25,060 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46166 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:54:25,060 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/C is initiating minor compaction (all files) 2024-11-20T16:54:25,061 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/C in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:25,061 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/5ae5757d30314a519fc4f0610239dd69, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/6864a74719a04d45a819054e9341e4c4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/17adca6e4b494dbeadeded6ea348bf31, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/bd272765bf0d4ae58e17f71a5352e6b3] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=45.1 K 2024-11-20T16:54:25,061 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ae5757d30314a519fc4f0610239dd69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732121658923 2024-11-20T16:54:25,061 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6864a74719a04d45a819054e9341e4c4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732121661173 2024-11-20T16:54:25,063 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17adca6e4b494dbeadeded6ea348bf31, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732121662421 2024-11-20T16:54:25,063 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd272765bf0d4ae58e17f71a5352e6b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732121663358 2024-11-20T16:54:25,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121725062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:25,065 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/B of 81890a01db534dee41b8dda5131aae4e into 1a2dc0af6da44336aaee7796f229988c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:25,065 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:25,066 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/B, priority=12, startTime=1732121664610; duration=0sec 2024-11-20T16:54:25,066 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:25,066 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:B 2024-11-20T16:54:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:25,072 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#C#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:25,073 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/f1792067e0b848d1b027ff89ba3b2dd3 is 50, key is test_row_0/C:col10/1732121663380/Put/seqid=0 2024-11-20T16:54:25,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:25,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121725066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:25,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121725066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:25,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742344_1520 (size=12493) 2024-11-20T16:54:25,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121725176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:25,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:25,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121725185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:25,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121725185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T16:54:25,336 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:25,337 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:54:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-20T16:54:25,339 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:54:25,339 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:54:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T16:54:25,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:54:25,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/2243e286cc1d444abfd9a620572181dc 2024-11-20T16:54:25,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/c0beb5e454e347f99ec0c2dba0fab33f is 50, key is test_row_0/B:col10/1732121663770/Put/seqid=0 2024-11-20T16:54:25,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742345_1521 (size=12151) 2024-11-20T16:54:25,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/c0beb5e454e347f99ec0c2dba0fab33f 2024-11-20T16:54:25,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/31fe1b5457c94ef686a2b2df59078cf0 is 50, key is test_row_0/C:col10/1732121663770/Put/seqid=0 2024-11-20T16:54:25,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742346_1522 (size=12151) 2024-11-20T16:54:25,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:25,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121725397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:25,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121725406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:25,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121725406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:25,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47156 deadline: 1732121725416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:25,422 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_0' in 81890a01db534dee41b8dda5131aae4e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T16:54:25,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-20T16:54:25,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:25,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:25,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:25,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:25,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:25,494 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/f1792067e0b848d1b027ff89ba3b2dd3 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/f1792067e0b848d1b027ff89ba3b2dd3 2024-11-20T16:54:25,499 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/C of 81890a01db534dee41b8dda5131aae4e into f1792067e0b848d1b027ff89ba3b2dd3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:25,499 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:25,499 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/C, priority=12, startTime=1732121664610; duration=0sec 2024-11-20T16:54:25,500 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:25,500 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:C 2024-11-20T16:54:25,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-20T16:54:25,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:25,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:25,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:25,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:25,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T16:54:25,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121725716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121725716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:25,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121725716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:25,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-20T16:54:25,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:25,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:25,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:25,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:25,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:25,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/31fe1b5457c94ef686a2b2df59078cf0 2024-11-20T16:54:25,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/2243e286cc1d444abfd9a620572181dc as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/2243e286cc1d444abfd9a620572181dc 2024-11-20T16:54:25,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/2243e286cc1d444abfd9a620572181dc, entries=200, sequenceid=241, filesize=14.2 K 2024-11-20T16:54:25,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/c0beb5e454e347f99ec0c2dba0fab33f as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/c0beb5e454e347f99ec0c2dba0fab33f 2024-11-20T16:54:25,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/c0beb5e454e347f99ec0c2dba0fab33f, entries=150, sequenceid=241, filesize=11.9 K 2024-11-20T16:54:25,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/31fe1b5457c94ef686a2b2df59078cf0 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/31fe1b5457c94ef686a2b2df59078cf0 2024-11-20T16:54:25,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/31fe1b5457c94ef686a2b2df59078cf0, entries=150, sequenceid=241, filesize=11.9 K 2024-11-20T16:54:25,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=127.47 KB/130530 for 81890a01db534dee41b8dda5131aae4e in 853ms, sequenceid=241, compaction requested=false 2024-11-20T16:54:25,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:25,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-20T16:54:25,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:25,952 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-20T16:54:25,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:25,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:25,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:25,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:25,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:25,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/831f0fccd12a45869feecd5478201fa1 is 50, key is test_row_0/A:col10/1732121665063/Put/seqid=0 2024-11-20T16:54:25,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T16:54:25,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742347_1523 (size=12301) 2024-11-20T16:54:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:26,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:26,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121726119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:26,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:26,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:26,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:26,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121726236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:26,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:26,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:26,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121726236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:26,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:26,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121726236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:26,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121726236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:26,368 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/831f0fccd12a45869feecd5478201fa1 2024-11-20T16:54:26,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/37ee42b28841450a875c4e12bf46e6f6 is 50, key is test_row_0/B:col10/1732121665063/Put/seqid=0 2024-11-20T16:54:26,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742348_1524 (size=12301) 2024-11-20T16:54:26,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121726446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T16:54:26,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121726756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:26,781 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/37ee42b28841450a875c4e12bf46e6f6 2024-11-20T16:54:26,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/2d6db689f4c540b7bfcdabfcbb6e0b6d is 50, key is test_row_0/C:col10/1732121665063/Put/seqid=0 2024-11-20T16:54:26,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742349_1525 (size=12301) 2024-11-20T16:54:27,188 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/2d6db689f4c540b7bfcdabfcbb6e0b6d 2024-11-20T16:54:27,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/831f0fccd12a45869feecd5478201fa1 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/831f0fccd12a45869feecd5478201fa1 2024-11-20T16:54:27,195 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/831f0fccd12a45869feecd5478201fa1, entries=150, sequenceid=266, filesize=12.0 K 2024-11-20T16:54:27,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/37ee42b28841450a875c4e12bf46e6f6 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/37ee42b28841450a875c4e12bf46e6f6 2024-11-20T16:54:27,198 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/37ee42b28841450a875c4e12bf46e6f6, entries=150, sequenceid=266, filesize=12.0 K 2024-11-20T16:54:27,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/2d6db689f4c540b7bfcdabfcbb6e0b6d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/2d6db689f4c540b7bfcdabfcbb6e0b6d 2024-11-20T16:54:27,201 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/2d6db689f4c540b7bfcdabfcbb6e0b6d, entries=150, sequenceid=266, filesize=12.0 K 2024-11-20T16:54:27,202 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 81890a01db534dee41b8dda5131aae4e in 1250ms, sequenceid=266, compaction requested=true 2024-11-20T16:54:27,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:27,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:27,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-20T16:54:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-20T16:54:27,205 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-20T16:54:27,205 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8640 sec 2024-11-20T16:54:27,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.8680 sec 2024-11-20T16:54:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:27,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T16:54:27,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:27,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:27,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:27,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:27,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:27,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:27,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/b7aa385651de487da2ef7f28473b6729 is 50, key is test_row_0/A:col10/1732121667262/Put/seqid=0 2024-11-20T16:54:27,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742350_1526 (size=12297) 2024-11-20T16:54:27,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121727289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:27,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121727290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:27,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121667284, max timestamp=1732121667299Num uniques 859; 2024-11-20T16:54:27,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121667284, max timestamp=1732121667299Num uniques 859; 2024-11-20T16:54:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:27,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121667284, max timestamp=1732121667299Num uniques 859; 2024-11-20T16:54:27,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:27,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:27,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121727355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:27,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121727364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:27,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:27,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121727405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:27,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:27,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121727406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:27,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:27,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T16:54:27,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121727476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:27,486 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:27,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:27,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121727476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:27,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:27,486 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:54:27,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-20T16:54:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T16:54:27,488 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:54:27,488 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:54:27,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:54:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T16:54:27,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121727616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:27,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121727616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:27,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T16:54:27,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:27,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:27,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:27,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:27,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:27,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/b7aa385651de487da2ef7f28473b6729 2024-11-20T16:54:27,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/09c58671a72c41d48dac29be71ee80eb is 50, key is test_row_0/B:col10/1732121667262/Put/seqid=0 2024-11-20T16:54:27,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742351_1527 (size=9857) 2024-11-20T16:54:27,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121727695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:27,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121727695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:27,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T16:54:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:27,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:27,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T16:54:27,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121727925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:27,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:27,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T16:54:27,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:27,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:27,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:27,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:27,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:27,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121727936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:27,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:27,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121728006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:28,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121728006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:28,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/09c58671a72c41d48dac29be71ee80eb 2024-11-20T16:54:28,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/221c51b330184dccaa1fb6e8a0ed7b96 is 50, key is test_row_0/C:col10/1732121667262/Put/seqid=0 2024-11-20T16:54:28,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742352_1528 (size=9857) 2024-11-20T16:54:28,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T16:54:28,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:28,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:28,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:28,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T16:54:28,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T16:54:28,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:28,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:28,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:28,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T16:54:28,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:28,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:28,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:28,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121728436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:28,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121728456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:28,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:28,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/221c51b330184dccaa1fb6e8a0ed7b96 2024-11-20T16:54:28,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/b7aa385651de487da2ef7f28473b6729 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/b7aa385651de487da2ef7f28473b6729 2024-11-20T16:54:28,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/b7aa385651de487da2ef7f28473b6729, entries=150, sequenceid=277, filesize=12.0 K 2024-11-20T16:54:28,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/09c58671a72c41d48dac29be71ee80eb as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/09c58671a72c41d48dac29be71ee80eb 2024-11-20T16:54:28,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/09c58671a72c41d48dac29be71ee80eb, entries=100, sequenceid=277, filesize=9.6 K 2024-11-20T16:54:28,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/221c51b330184dccaa1fb6e8a0ed7b96 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/221c51b330184dccaa1fb6e8a0ed7b96 2024-11-20T16:54:28,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/221c51b330184dccaa1fb6e8a0ed7b96, entries=100, sequenceid=277, filesize=9.6 K 2024-11-20T16:54:28,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 81890a01db534dee41b8dda5131aae4e in 1235ms, sequenceid=277, compaction requested=true 2024-11-20T16:54:28,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:28,498 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:54:28,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:54:28,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:28,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:54:28,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:28,498 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:54:28,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:54:28,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:28,499 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46802 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:54:28,499 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51632 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:54:28,499 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/A is initiating minor compaction (all files) 2024-11-20T16:54:28,499 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/B is initiating minor compaction (all files) 2024-11-20T16:54:28,499 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/B in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:28,499 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/A in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:28,499 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/badf0a45358647aaa08b3b849acf4714, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/2243e286cc1d444abfd9a620572181dc, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/831f0fccd12a45869feecd5478201fa1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/b7aa385651de487da2ef7f28473b6729] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=50.4 K 2024-11-20T16:54:28,499 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/1a2dc0af6da44336aaee7796f229988c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/c0beb5e454e347f99ec0c2dba0fab33f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/37ee42b28841450a875c4e12bf46e6f6, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/09c58671a72c41d48dac29be71ee80eb] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=45.7 K 2024-11-20T16:54:28,500 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting badf0a45358647aaa08b3b849acf4714, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732121663358 2024-11-20T16:54:28,500 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a2dc0af6da44336aaee7796f229988c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732121663358 2024-11-20T16:54:28,500 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting c0beb5e454e347f99ec0c2dba0fab33f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732121663770 2024-11-20T16:54:28,500 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2243e286cc1d444abfd9a620572181dc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732121663750 2024-11-20T16:54:28,500 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 37ee42b28841450a875c4e12bf46e6f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732121665060 2024-11-20T16:54:28,500 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 831f0fccd12a45869feecd5478201fa1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732121665060 2024-11-20T16:54:28,500 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 09c58671a72c41d48dac29be71ee80eb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732121666103 2024-11-20T16:54:28,500 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7aa385651de487da2ef7f28473b6729, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732121666103 2024-11-20T16:54:28,507 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#A#compaction#442 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:28,507 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#B#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:28,507 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/fe86cf5066304c92979695af3cbe60c4 is 50, key is test_row_0/A:col10/1732121667262/Put/seqid=0 2024-11-20T16:54:28,508 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/a9954e75f83b4e58a957038453ae5aa0 is 50, key is test_row_0/B:col10/1732121667262/Put/seqid=0 2024-11-20T16:54:28,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742354_1530 (size=12779) 2024-11-20T16:54:28,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742353_1529 (size=12779) 2024-11-20T16:54:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:28,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-20T16:54:28,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:28,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:28,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:28,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:28,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:28,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:28,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/93b6a269daf24081957b2960ebe270b5 is 50, key is test_row_0/A:col10/1732121667354/Put/seqid=0 2024-11-20T16:54:28,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742355_1531 (size=14741) 2024-11-20T16:54:28,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121728557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:28,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121728567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:28,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:28,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T16:54:28,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:28,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:28,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:28,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T16:54:28,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121728686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:28,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121728686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:28,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T16:54:28,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:28,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:28,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:28,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T16:54:28,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:28,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:28,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:28,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:28,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121728896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:28,916 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/fe86cf5066304c92979695af3cbe60c4 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/fe86cf5066304c92979695af3cbe60c4 2024-11-20T16:54:28,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121728906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:28,920 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/A of 81890a01db534dee41b8dda5131aae4e into fe86cf5066304c92979695af3cbe60c4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:28,920 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:28,920 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/A, priority=12, startTime=1732121668498; duration=0sec 2024-11-20T16:54:28,920 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:28,920 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:A 2024-11-20T16:54:28,920 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:54:28,921 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46802 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:54:28,921 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/C is initiating minor compaction (all files) 2024-11-20T16:54:28,921 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/C in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:28,921 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/f1792067e0b848d1b027ff89ba3b2dd3, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/31fe1b5457c94ef686a2b2df59078cf0, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/2d6db689f4c540b7bfcdabfcbb6e0b6d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/221c51b330184dccaa1fb6e8a0ed7b96] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=45.7 K 2024-11-20T16:54:28,922 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1792067e0b848d1b027ff89ba3b2dd3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732121663358 2024-11-20T16:54:28,922 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31fe1b5457c94ef686a2b2df59078cf0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732121663770 2024-11-20T16:54:28,922 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d6db689f4c540b7bfcdabfcbb6e0b6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732121665060 2024-11-20T16:54:28,922 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 221c51b330184dccaa1fb6e8a0ed7b96, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732121666103 2024-11-20T16:54:28,922 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/a9954e75f83b4e58a957038453ae5aa0 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/a9954e75f83b4e58a957038453ae5aa0 2024-11-20T16:54:28,926 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/B of 81890a01db534dee41b8dda5131aae4e into a9954e75f83b4e58a957038453ae5aa0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:28,926 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:28,926 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/B, priority=12, startTime=1732121668498; duration=0sec 2024-11-20T16:54:28,926 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:28,926 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:B 2024-11-20T16:54:28,928 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#C#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:28,928 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/5614931662694cf3bc9331376f96b057 is 50, key is test_row_0/C:col10/1732121667262/Put/seqid=0 2024-11-20T16:54:28,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742356_1532 (size=12779) 2024-11-20T16:54:28,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/93b6a269daf24081957b2960ebe270b5 2024-11-20T16:54:28,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/623734bdf1244b8884b953e621f6644a is 50, key is test_row_0/B:col10/1732121667354/Put/seqid=0 2024-11-20T16:54:28,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742357_1533 (size=12301) 2024-11-20T16:54:29,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T16:54:29,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:29,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:29,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:29,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:29,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:29,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T16:54:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:29,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:29,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121729216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:29,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:29,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:29,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:29,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121729225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:29,334 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/5614931662694cf3bc9331376f96b057 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/5614931662694cf3bc9331376f96b057 2024-11-20T16:54:29,338 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/C of 81890a01db534dee41b8dda5131aae4e into 5614931662694cf3bc9331376f96b057(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:29,338 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:29,338 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/C, priority=12, startTime=1732121668498; duration=0sec 2024-11-20T16:54:29,338 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:29,338 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:C 2024-11-20T16:54:29,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/623734bdf1244b8884b953e621f6644a 2024-11-20T16:54:29,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T16:54:29,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:29,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:29,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:29,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:29,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:29,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/d20c124054894289a8963ba5dda2c77f is 50, key is test_row_0/C:col10/1732121667354/Put/seqid=0 2024-11-20T16:54:29,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742358_1534 (size=12301) 2024-11-20T16:54:29,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47156 deadline: 1732121729446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:29,454 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_0' in 81890a01db534dee41b8dda5131aae4e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:29,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121729456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:29,466 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 81890a01db534dee41b8dda5131aae4e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:29,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:29,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:29,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:29,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121729476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:29,480 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 81890a01db534dee41b8dda5131aae4e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:29,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:29,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:29,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T16:54:29,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:29,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:29,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:29,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:29,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T16:54:29,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T16:54:29,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:29,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:29,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:29,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:29,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:29,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121729725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:29,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121729746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:29,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:29,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:29,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:29,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/d20c124054894289a8963ba5dda2c77f 2024-11-20T16:54:29,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/93b6a269daf24081957b2960ebe270b5 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/93b6a269daf24081957b2960ebe270b5 2024-11-20T16:54:29,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/93b6a269daf24081957b2960ebe270b5, entries=200, sequenceid=313, filesize=14.4 K 2024-11-20T16:54:29,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/623734bdf1244b8884b953e621f6644a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/623734bdf1244b8884b953e621f6644a 2024-11-20T16:54:29,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/623734bdf1244b8884b953e621f6644a, entries=150, sequenceid=313, filesize=12.0 K 2024-11-20T16:54:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/d20c124054894289a8963ba5dda2c77f as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/d20c124054894289a8963ba5dda2c77f 2024-11-20T16:54:29,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/d20c124054894289a8963ba5dda2c77f, entries=150, sequenceid=313, filesize=12.0 K 2024-11-20T16:54:29,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 81890a01db534dee41b8dda5131aae4e in 1250ms, sequenceid=313, compaction requested=false 2024-11-20T16:54:29,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:29,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T16:54:29,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:29,816 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T16:54:29,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:29,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:29,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:29,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:29,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:29,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:29,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/835475c8335e4c829a155c891db133fb is 50, key is test_row_0/A:col10/1732121668566/Put/seqid=0 2024-11-20T16:54:29,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742359_1535 (size=9857) 2024-11-20T16:54:30,226 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/835475c8335e4c829a155c891db133fb 2024-11-20T16:54:30,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/03c3e9d19aa8408e997574e5a043b51b is 50, key is test_row_0/B:col10/1732121668566/Put/seqid=0 2024-11-20T16:54:30,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742360_1536 (size=9857) 2024-11-20T16:54:30,634 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/03c3e9d19aa8408e997574e5a043b51b 2024-11-20T16:54:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/184bbd39ecdd489f9381bd77bee9876c is 50, key is test_row_0/C:col10/1732121668566/Put/seqid=0 2024-11-20T16:54:30,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742361_1537 (size=9857) 2024-11-20T16:54:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:30,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:30,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121670775, max timestamp=1732121670809Num uniques 859; 2024-11-20T16:54:30,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121670775, max timestamp=1732121670809Num uniques 859; 2024-11-20T16:54:30,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121670775, max timestamp=1732121670809Num uniques 859; 2024-11-20T16:54:30,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:30,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:30,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:30,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121730874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:30,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121730875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:30,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:30,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:30,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:30,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:31,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121730996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:31,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:31,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121730996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:31,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:31,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:31,049 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/184bbd39ecdd489f9381bd77bee9876c 2024-11-20T16:54:31,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/835475c8335e4c829a155c891db133fb as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/835475c8335e4c829a155c891db133fb 2024-11-20T16:54:31,055 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/835475c8335e4c829a155c891db133fb, entries=100, sequenceid=326, filesize=9.6 K 2024-11-20T16:54:31,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/03c3e9d19aa8408e997574e5a043b51b as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/03c3e9d19aa8408e997574e5a043b51b 2024-11-20T16:54:31,057 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/03c3e9d19aa8408e997574e5a043b51b, entries=100, sequenceid=326, filesize=9.6 K 2024-11-20T16:54:31,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/184bbd39ecdd489f9381bd77bee9876c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/184bbd39ecdd489f9381bd77bee9876c 2024-11-20T16:54:31,060 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/184bbd39ecdd489f9381bd77bee9876c, entries=100, sequenceid=326, filesize=9.6 K 2024-11-20T16:54:31,061 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 81890a01db534dee41b8dda5131aae4e in 1245ms, sequenceid=326, compaction requested=true 2024-11-20T16:54:31,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:31,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:31,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-20T16:54:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-20T16:54:31,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-20T16:54:31,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5740 sec 2024-11-20T16:54:31,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 3.5780 sec 2024-11-20T16:54:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:31,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-20T16:54:31,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:31,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:31,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:31,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:31,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:31,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:31,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/96bc30758bfd4b98bd898b91e9c2d50a is 50, key is test_row_0/A:col10/1732121670867/Put/seqid=0 2024-11-20T16:54:31,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742362_1538 (size=14741) 2024-11-20T16:54:31,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121731241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:31,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121731242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:31,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121731356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:31,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:31,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121731356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:31,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:31,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121731475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:31,480 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 81890a01db534dee41b8dda5131aae4e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:31,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121731496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:31,506 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 81890a01db534dee41b8dda5131aae4e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:31,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:31,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121731566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:31,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:31,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121731566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:31,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:31,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:31,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/96bc30758bfd4b98bd898b91e9c2d50a 2024-11-20T16:54:31,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/c9b409364b5646a6bda519b792714afa is 50, key is test_row_0/B:col10/1732121670867/Put/seqid=0 2024-11-20T16:54:31,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742363_1539 (size=12301) 2024-11-20T16:54:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T16:54:31,657 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:31,657 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:54:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-20T16:54:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T16:54:31,659 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:54:31,659 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:54:31,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:54:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T16:54:31,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-20T16:54:31,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:31,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:31,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:31,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:31,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:31,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:31,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121731886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:31,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121731886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:31,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-20T16:54:31,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:31,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:31,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:31,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:31,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:31,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:31,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T16:54:32,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/c9b409364b5646a6bda519b792714afa 2024-11-20T16:54:32,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/de1df42d82c142a695d476616ca793ad is 50, key is test_row_0/C:col10/1732121670867/Put/seqid=0 2024-11-20T16:54:32,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742364_1540 (size=12301) 2024-11-20T16:54:32,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-20T16:54:32,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:32,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:32,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:32,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:32,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:32,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-20T16:54:32,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:32,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:32,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:32,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:32,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:32,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:32,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T16:54:32,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121732406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:32,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121732406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:32,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:32,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:32,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:32,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:32,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:32,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:32,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-20T16:54:32,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:32,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:32,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:32,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:32,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:32,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/de1df42d82c142a695d476616ca793ad 2024-11-20T16:54:32,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/96bc30758bfd4b98bd898b91e9c2d50a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/96bc30758bfd4b98bd898b91e9c2d50a 2024-11-20T16:54:32,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/96bc30758bfd4b98bd898b91e9c2d50a, entries=200, sequenceid=363, filesize=14.4 K 2024-11-20T16:54:32,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/c9b409364b5646a6bda519b792714afa as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/c9b409364b5646a6bda519b792714afa 2024-11-20T16:54:32,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/c9b409364b5646a6bda519b792714afa, entries=150, sequenceid=363, filesize=12.0 K 2024-11-20T16:54:32,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/de1df42d82c142a695d476616ca793ad as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/de1df42d82c142a695d476616ca793ad 2024-11-20T16:54:32,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/de1df42d82c142a695d476616ca793ad, entries=150, sequenceid=363, filesize=12.0 K 2024-11-20T16:54:32,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 81890a01db534dee41b8dda5131aae4e in 1237ms, sequenceid=363, compaction requested=true 2024-11-20T16:54:32,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:32,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:54:32,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:32,448 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:54:32,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:54:32,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:32,448 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:54:32,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:54:32,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:32,449 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52118 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:54:32,449 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47238 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:54:32,449 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/B is initiating minor compaction (all files) 2024-11-20T16:54:32,449 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/A is initiating minor compaction (all files) 2024-11-20T16:54:32,449 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/B in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:32,449 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/A in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:32,449 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/a9954e75f83b4e58a957038453ae5aa0, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/623734bdf1244b8884b953e621f6644a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/03c3e9d19aa8408e997574e5a043b51b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/c9b409364b5646a6bda519b792714afa] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=46.1 K 2024-11-20T16:54:32,449 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/fe86cf5066304c92979695af3cbe60c4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/93b6a269daf24081957b2960ebe270b5, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/835475c8335e4c829a155c891db133fb, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/96bc30758bfd4b98bd898b91e9c2d50a] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=50.9 K 2024-11-20T16:54:32,449 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting a9954e75f83b4e58a957038453ae5aa0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732121665060 2024-11-20T16:54:32,449 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe86cf5066304c92979695af3cbe60c4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732121665060 2024-11-20T16:54:32,450 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93b6a269daf24081957b2960ebe270b5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732121667349 2024-11-20T16:54:32,450 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 623734bdf1244b8884b953e621f6644a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732121667349 2024-11-20T16:54:32,450 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 835475c8335e4c829a155c891db133fb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732121668555 2024-11-20T16:54:32,450 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 03c3e9d19aa8408e997574e5a043b51b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732121668555 2024-11-20T16:54:32,450 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96bc30758bfd4b98bd898b91e9c2d50a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732121670857 2024-11-20T16:54:32,450 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting c9b409364b5646a6bda519b792714afa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732121670857 2024-11-20T16:54:32,456 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#B#compaction#454 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:32,456 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#A#compaction#455 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:32,457 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/9f59c3772a294012a5eb99b643e8c2af is 50, key is test_row_0/A:col10/1732121670867/Put/seqid=0 2024-11-20T16:54:32,457 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/1cf954f78fac4e09bf899676a88eae3f is 50, key is test_row_0/B:col10/1732121670867/Put/seqid=0 2024-11-20T16:54:32,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742366_1542 (size=12915) 2024-11-20T16:54:32,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742365_1541 (size=12915) 2024-11-20T16:54:32,468 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/1cf954f78fac4e09bf899676a88eae3f as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/1cf954f78fac4e09bf899676a88eae3f 2024-11-20T16:54:32,471 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/B of 81890a01db534dee41b8dda5131aae4e into 1cf954f78fac4e09bf899676a88eae3f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:32,471 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:32,472 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/B, priority=12, startTime=1732121672448; duration=0sec 2024-11-20T16:54:32,472 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:32,472 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:B 2024-11-20T16:54:32,472 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:54:32,472 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47238 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:54:32,473 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/C is initiating minor compaction (all files) 2024-11-20T16:54:32,473 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/C in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:32,473 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/5614931662694cf3bc9331376f96b057, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/d20c124054894289a8963ba5dda2c77f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/184bbd39ecdd489f9381bd77bee9876c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/de1df42d82c142a695d476616ca793ad] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=46.1 K 2024-11-20T16:54:32,473 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 5614931662694cf3bc9331376f96b057, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732121665060 2024-11-20T16:54:32,473 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting d20c124054894289a8963ba5dda2c77f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732121667349 2024-11-20T16:54:32,473 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 184bbd39ecdd489f9381bd77bee9876c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732121668555 2024-11-20T16:54:32,473 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting de1df42d82c142a695d476616ca793ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732121670857 2024-11-20T16:54:32,479 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#C#compaction#456 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:32,479 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/1863135083bf490c9b5663a9a4f5a805 is 50, key is test_row_0/C:col10/1732121670867/Put/seqid=0 2024-11-20T16:54:32,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742367_1543 (size=12915) 2024-11-20T16:54:32,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-20T16:54:32,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:32,574 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-20T16:54:32,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:32,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:32,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:32,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:32,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:32,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:32,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/4d31364fd7604a7ea538a26d3aae1a73 is 50, key is test_row_0/A:col10/1732121671226/Put/seqid=0 2024-11-20T16:54:32,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742368_1544 (size=12301) 2024-11-20T16:54:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T16:54:32,867 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/9f59c3772a294012a5eb99b643e8c2af as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/9f59c3772a294012a5eb99b643e8c2af 2024-11-20T16:54:32,870 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/A of 81890a01db534dee41b8dda5131aae4e into 9f59c3772a294012a5eb99b643e8c2af(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:32,870 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:32,871 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/A, priority=12, startTime=1732121672448; duration=0sec 2024-11-20T16:54:32,871 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:32,871 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:A 2024-11-20T16:54:32,885 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/1863135083bf490c9b5663a9a4f5a805 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/1863135083bf490c9b5663a9a4f5a805 2024-11-20T16:54:32,888 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/C of 81890a01db534dee41b8dda5131aae4e into 1863135083bf490c9b5663a9a4f5a805(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:32,888 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:32,888 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/C, priority=12, startTime=1732121672448; duration=0sec 2024-11-20T16:54:32,888 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:32,888 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:C 2024-11-20T16:54:32,981 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/4d31364fd7604a7ea538a26d3aae1a73 2024-11-20T16:54:32,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/b84534b22f7b452086c7e8424c6a966a is 50, key is test_row_0/B:col10/1732121671226/Put/seqid=0 2024-11-20T16:54:32,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742369_1545 (size=12301) 2024-11-20T16:54:33,389 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/b84534b22f7b452086c7e8424c6a966a 2024-11-20T16:54:33,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:33,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:33,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/15d67ec37b1c420cbaa1955c8bf43f3e is 50, key is test_row_0/C:col10/1732121671226/Put/seqid=0 2024-11-20T16:54:33,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742370_1546 (size=12301) 2024-11-20T16:54:33,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121673457, max timestamp=1732121673535Num uniques 859; 2024-11-20T16:54:33,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121673457, max timestamp=1732121673535Num uniques 859; 2024-11-20T16:54:33,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121673457, max timestamp=1732121673535Num uniques 859; 2024-11-20T16:54:33,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:33,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:33,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:33,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121733608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:33,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121733612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:33,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121733726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:33,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121733726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:33,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:33,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:33,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:33,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:33,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:33,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:33,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T16:54:33,845 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/15d67ec37b1c420cbaa1955c8bf43f3e 2024-11-20T16:54:33,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/4d31364fd7604a7ea538a26d3aae1a73 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/4d31364fd7604a7ea538a26d3aae1a73 2024-11-20T16:54:33,851 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/4d31364fd7604a7ea538a26d3aae1a73, entries=150, sequenceid=373, filesize=12.0 K 2024-11-20T16:54:33,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/b84534b22f7b452086c7e8424c6a966a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/b84534b22f7b452086c7e8424c6a966a 2024-11-20T16:54:33,854 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/b84534b22f7b452086c7e8424c6a966a, entries=150, sequenceid=373, filesize=12.0 K 2024-11-20T16:54:33,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/15d67ec37b1c420cbaa1955c8bf43f3e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/15d67ec37b1c420cbaa1955c8bf43f3e 2024-11-20T16:54:33,857 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/15d67ec37b1c420cbaa1955c8bf43f3e, entries=150, sequenceid=373, filesize=12.0 K 2024-11-20T16:54:33,858 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 81890a01db534dee41b8dda5131aae4e in 1284ms, sequenceid=373, compaction requested=false 2024-11-20T16:54:33,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:33,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:33,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-20T16:54:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-20T16:54:33,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-20T16:54:33,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2000 sec 2024-11-20T16:54:33,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 2.2040 sec 2024-11-20T16:54:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:33,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-20T16:54:33,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:33,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T16:54:33,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:33,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T16:54:33,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:33,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T16:54:33,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/6d8ef5e28480437dafb13cccba60126c is 50, key is test_row_0/A:col10/1732121673597/Put/seqid=0 2024-11-20T16:54:33,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742371_1547 (size=17181) 2024-11-20T16:54:33,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:33,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121733959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:33,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:33,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121733963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:34,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:34,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121734076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:34,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121734076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:34,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:34,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121734296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:34,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:34,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:34,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:34,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121734296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:34,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:34,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:34,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:34,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:34,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/6d8ef5e28480437dafb13cccba60126c 2024-11-20T16:54:34,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/2386e450cbad46f79b0e967c963f64b2 is 50, key is test_row_0/B:col10/1732121673597/Put/seqid=0 2024-11-20T16:54:34,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742372_1548 (size=12301) 2024-11-20T16:54:34,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/2386e450cbad46f79b0e967c963f64b2 2024-11-20T16:54:34,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/e5ea2d80eb40427da6bd502565033e2f is 50, key is test_row_0/C:col10/1732121673597/Put/seqid=0 2024-11-20T16:54:34,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742373_1549 (size=12301) 2024-11-20T16:54:34,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121734615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:34,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121734615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:34,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:34,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:34,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:34,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:34,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:34,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:34,674 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T16:54:34,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/e5ea2d80eb40427da6bd502565033e2f 2024-11-20T16:54:34,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/6d8ef5e28480437dafb13cccba60126c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/6d8ef5e28480437dafb13cccba60126c 2024-11-20T16:54:34,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/6d8ef5e28480437dafb13cccba60126c, entries=250, sequenceid=414, filesize=16.8 K 2024-11-20T16:54:34,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/2386e450cbad46f79b0e967c963f64b2 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/2386e450cbad46f79b0e967c963f64b2 2024-11-20T16:54:34,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/2386e450cbad46f79b0e967c963f64b2, entries=150, sequenceid=414, filesize=12.0 K 2024-11-20T16:54:34,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/e5ea2d80eb40427da6bd502565033e2f as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/e5ea2d80eb40427da6bd502565033e2f 2024-11-20T16:54:34,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/e5ea2d80eb40427da6bd502565033e2f, entries=150, sequenceid=414, filesize=12.0 K 2024-11-20T16:54:34,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~440.72 KB/451296, currentSize=26.84 KB/27480 for 81890a01db534dee41b8dda5131aae4e in 853ms, sequenceid=414, compaction requested=true 2024-11-20T16:54:34,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:34,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:54:34,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:34,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:54:34,796 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:54:34,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:34,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:54:34,796 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:54:34,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:34,797 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42397 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:54:34,797 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:54:34,797 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/B is initiating minor compaction (all files) 2024-11-20T16:54:34,797 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/A is initiating minor compaction (all files) 2024-11-20T16:54:34,797 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/A in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:34,797 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/B in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:34,797 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/1cf954f78fac4e09bf899676a88eae3f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/b84534b22f7b452086c7e8424c6a966a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/2386e450cbad46f79b0e967c963f64b2] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=36.6 K 2024-11-20T16:54:34,797 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/9f59c3772a294012a5eb99b643e8c2af, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/4d31364fd7604a7ea538a26d3aae1a73, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/6d8ef5e28480437dafb13cccba60126c] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=41.4 K 2024-11-20T16:54:34,798 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f59c3772a294012a5eb99b643e8c2af, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732121670857 2024-11-20T16:54:34,798 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cf954f78fac4e09bf899676a88eae3f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732121670857 2024-11-20T16:54:34,798 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting b84534b22f7b452086c7e8424c6a966a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732121671226 2024-11-20T16:54:34,798 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d31364fd7604a7ea538a26d3aae1a73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732121671226 2024-11-20T16:54:34,798 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 2386e450cbad46f79b0e967c963f64b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732121673597 2024-11-20T16:54:34,798 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d8ef5e28480437dafb13cccba60126c, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732121673597 2024-11-20T16:54:34,803 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#A#compaction#463 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:34,803 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/36e4139d51434ed4a884a27bcdbb1b7b is 50, key is test_row_0/A:col10/1732121673597/Put/seqid=0 2024-11-20T16:54:34,806 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#B#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:34,806 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/24593330381c41ed992e8c1b5a258fc8 is 50, key is test_row_0/B:col10/1732121673597/Put/seqid=0 2024-11-20T16:54:34,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742374_1550 (size=13017) 2024-11-20T16:54:34,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742375_1551 (size=13017) 2024-11-20T16:54:34,814 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/36e4139d51434ed4a884a27bcdbb1b7b as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/36e4139d51434ed4a884a27bcdbb1b7b 2024-11-20T16:54:34,817 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/24593330381c41ed992e8c1b5a258fc8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/24593330381c41ed992e8c1b5a258fc8 2024-11-20T16:54:34,819 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/A of 81890a01db534dee41b8dda5131aae4e into 36e4139d51434ed4a884a27bcdbb1b7b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:34,819 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:34,819 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/A, priority=13, startTime=1732121674796; duration=0sec 2024-11-20T16:54:34,819 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:34,819 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:A 2024-11-20T16:54:34,819 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:54:34,820 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:54:34,820 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/C is initiating minor compaction (all files) 2024-11-20T16:54:34,820 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/C in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:34,820 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/1863135083bf490c9b5663a9a4f5a805, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/15d67ec37b1c420cbaa1955c8bf43f3e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/e5ea2d80eb40427da6bd502565033e2f] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=36.6 K 2024-11-20T16:54:34,821 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1863135083bf490c9b5663a9a4f5a805, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732121670857 2024-11-20T16:54:34,821 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/B of 81890a01db534dee41b8dda5131aae4e into 24593330381c41ed992e8c1b5a258fc8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:34,821 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:34,821 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15d67ec37b1c420cbaa1955c8bf43f3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732121671226 2024-11-20T16:54:34,821 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/B, priority=13, startTime=1732121674796; duration=0sec 2024-11-20T16:54:34,821 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:34,821 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:B 2024-11-20T16:54:34,821 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5ea2d80eb40427da6bd502565033e2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732121673597 2024-11-20T16:54:34,827 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#C#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:34,827 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/808bf5b28d094326a90f54fc7fc5d445 is 50, key is test_row_0/C:col10/1732121673597/Put/seqid=0 2024-11-20T16:54:34,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742376_1552 (size=13017) 2024-11-20T16:54:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:35,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T16:54:35,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:35,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:35,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:35,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:35,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:35,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:35,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/b5648455d1c44b989b98f6273733d38d is 50, key is test_row_0/A:col10/1732121675138/Put/seqid=0 2024-11-20T16:54:35,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742377_1553 (size=17181) 2024-11-20T16:54:35,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121675181, max timestamp=1732121675207Num uniques 859; 2024-11-20T16:54:35,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:35,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121675181, max timestamp=1732121675207Num uniques 859; 2024-11-20T16:54:35,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:35,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121675181, max timestamp=1732121675207Num uniques 859; 2024-11-20T16:54:35,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:35,234 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/808bf5b28d094326a90f54fc7fc5d445 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/808bf5b28d094326a90f54fc7fc5d445 2024-11-20T16:54:35,236 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/C of 81890a01db534dee41b8dda5131aae4e into 808bf5b28d094326a90f54fc7fc5d445(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:35,236 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:35,236 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/C, priority=13, startTime=1732121674796; duration=0sec 2024-11-20T16:54:35,237 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:35,237 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:C 2024-11-20T16:54:35,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121735269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:35,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121735272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:35,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121735386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:35,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121735386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:35,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:35,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:35,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47174 deadline: 1732121735506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:35,509 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 81890a01db534dee41b8dda5131aae4e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:35,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/b5648455d1c44b989b98f6273733d38d 2024-11-20T16:54:35,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/2185ebfb4c624689bff7cfd9f6411074 is 50, key is test_row_0/B:col10/1732121675138/Put/seqid=0 2024-11-20T16:54:35,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47144 deadline: 1732121735546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:35,553 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 81890a01db534dee41b8dda5131aae4e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:35,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:35,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742378_1554 (size=12301) 2024-11-20T16:54:35,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121735596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:35,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121735596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T16:54:35,816 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:35,817 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:54:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees 2024-11-20T16:54:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-20T16:54:35,818 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:54:35,819 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:54:35,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:54:35,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121735916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:35,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:35,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121735916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:35,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:35,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:35,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-20T16:54:35,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/2185ebfb4c624689bff7cfd9f6411074 2024-11-20T16:54:35,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/e9cab47644654cabb96d8d0a884a2ef3 is 50, key is test_row_0/C:col10/1732121675138/Put/seqid=0 2024-11-20T16:54:35,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742379_1555 (size=12301) 2024-11-20T16:54:35,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T16:54:35,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:35,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:35,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:35,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:35,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:36,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T16:54:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:36,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-20T16:54:36,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T16:54:36,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:36,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:36,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:36,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:36,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:36,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/e9cab47644654cabb96d8d0a884a2ef3 2024-11-20T16:54:36,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/b5648455d1c44b989b98f6273733d38d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/b5648455d1c44b989b98f6273733d38d 2024-11-20T16:54:36,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/b5648455d1c44b989b98f6273733d38d, entries=250, sequenceid=426, filesize=16.8 K 2024-11-20T16:54:36,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/2185ebfb4c624689bff7cfd9f6411074 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/2185ebfb4c624689bff7cfd9f6411074 2024-11-20T16:54:36,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/2185ebfb4c624689bff7cfd9f6411074, entries=150, sequenceid=426, filesize=12.0 K 2024-11-20T16:54:36,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/e9cab47644654cabb96d8d0a884a2ef3 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/e9cab47644654cabb96d8d0a884a2ef3 2024-11-20T16:54:36,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/e9cab47644654cabb96d8d0a884a2ef3, entries=150, sequenceid=426, filesize=12.0 K 2024-11-20T16:54:36,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 81890a01db534dee41b8dda5131aae4e in 1237ms, sequenceid=426, compaction requested=false 2024-11-20T16:54:36,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:36,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T16:54:36,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:36,429 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-20T16:54:36,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:36,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:36,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:36,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:36,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:36,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:36,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/c9cdbae5aec54431954b3bdd1644e868 is 50, key is test_row_0/A:col10/1732121675246/Put/seqid=0 2024-11-20T16:54:36,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742380_1556 (size=12301) 2024-11-20T16:54:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:36,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. as already flushing 2024-11-20T16:54:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-20T16:54:36,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121736460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:36,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121736465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:36,551 INFO [Thread-2173 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2173. 2024-11-20T16:54:36,551 DEBUG [Thread-2173 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:36,551 DEBUG [Thread-2173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:36,552 DEBUG [Thread-2173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:36,552 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:36,554 INFO [Thread-2171 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2171. 2024-11-20T16:54:36,554 DEBUG [Thread-2171 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:36,554 DEBUG [Thread-2171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:36,554 DEBUG [Thread-2171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:36,554 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:36,558 INFO [Thread-2169 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2169. 2024-11-20T16:54:36,558 DEBUG [Thread-2169 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:36,558 DEBUG [Thread-2169 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:36,558 DEBUG [Thread-2169 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:36,558 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:36,566 INFO [Thread-2167 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2167. 2024-11-20T16:54:36,566 DEBUG [Thread-2167 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:36,566 DEBUG [Thread-2167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:36,566 DEBUG [Thread-2167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:36,566 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:36,567 INFO [Thread-2175 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2175. 2024-11-20T16:54:36,567 DEBUG [Thread-2175 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:36,567 DEBUG [Thread-2175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:36,567 DEBUG [Thread-2175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:36,567 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:36,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:36,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121736586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121736586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:36,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:36,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:36,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:36,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:36,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:36,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121736797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121736797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:36,838 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/c9cdbae5aec54431954b3bdd1644e868 2024-11-20T16:54:36,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/f0a01a94c92a4a7eaed467982e60994e is 50, key is test_row_0/B:col10/1732121675246/Put/seqid=0 2024-11-20T16:54:36,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742381_1557 (size=12301) 2024-11-20T16:54:36,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-20T16:54:37,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:37,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121737106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121737107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:37,254 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/f0a01a94c92a4a7eaed467982e60994e 2024-11-20T16:54:37,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/405166604b6f4b139d997cba00cdf42a is 50, key is test_row_0/C:col10/1732121675246/Put/seqid=0 2024-11-20T16:54:37,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742382_1558 (size=12301) 2024-11-20T16:54:37,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:37,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:37,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47202 deadline: 1732121737616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:37,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47196 deadline: 1732121737616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 , the old value is region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., hostname=b60eac9bda43,33929,1732121526384, seqNum=2 because the exception is null or not the one we care about 2024-11-20T16:54:37,669 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/405166604b6f4b139d997cba00cdf42a 2024-11-20T16:54:37,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/c9cdbae5aec54431954b3bdd1644e868 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/c9cdbae5aec54431954b3bdd1644e868 2024-11-20T16:54:37,677 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/c9cdbae5aec54431954b3bdd1644e868, entries=150, sequenceid=463, filesize=12.0 K 2024-11-20T16:54:37,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/f0a01a94c92a4a7eaed467982e60994e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/f0a01a94c92a4a7eaed467982e60994e 2024-11-20T16:54:37,681 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/f0a01a94c92a4a7eaed467982e60994e, entries=150, sequenceid=463, filesize=12.0 K 2024-11-20T16:54:37,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/405166604b6f4b139d997cba00cdf42a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/405166604b6f4b139d997cba00cdf42a 2024-11-20T16:54:37,686 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/405166604b6f4b139d997cba00cdf42a, entries=150, sequenceid=463, filesize=12.0 K 2024-11-20T16:54:37,687 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 81890a01db534dee41b8dda5131aae4e in 1258ms, sequenceid=463, compaction requested=true 2024-11-20T16:54:37,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:37,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:37,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-11-20T16:54:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=146 2024-11-20T16:54:37,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-20T16:54:37,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8700 sec 2024-11-20T16:54:37,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees in 1.8750 sec 2024-11-20T16:54:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-20T16:54:37,968 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:38,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T16:54:38,631 INFO [Thread-2164 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2164. 2024-11-20T16:54:38,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:38,631 DEBUG [Thread-2164 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:38,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:38,631 DEBUG [Thread-2164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:38,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:38,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:38,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:38,631 DEBUG [Thread-2164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:38,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:38,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:38,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/0ed97973d0ca41daaf7d6d7161191a85 is 50, key is test_row_0/A:col10/1732121676451/Put/seqid=0 2024-11-20T16:54:38,638 INFO [Thread-2158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2158. 2024-11-20T16:54:38,638 DEBUG [Thread-2158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:38,638 DEBUG [Thread-2158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:38,639 DEBUG [Thread-2158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:38,639 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:38,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742383_1559 (size=12301) 2024-11-20T16:54:39,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/0ed97973d0ca41daaf7d6d7161191a85 2024-11-20T16:54:39,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/ba5f236296dc4e72b2d2bdc874be7188 is 50, key is test_row_0/B:col10/1732121676451/Put/seqid=0 2024-11-20T16:54:39,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742384_1560 (size=12301) 2024-11-20T16:54:39,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/ba5f236296dc4e72b2d2bdc874be7188 2024-11-20T16:54:39,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/a28ae47130bc4a5eb343bf9ccc9b9398 is 50, key is test_row_0/C:col10/1732121676451/Put/seqid=0 2024-11-20T16:54:39,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742385_1561 (size=12301) 2024-11-20T16:54:39,560 INFO [Thread-2160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2160. 2024-11-20T16:54:39,560 DEBUG [Thread-2160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:39,560 DEBUG [Thread-2160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:39,560 DEBUG [Thread-2160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:39,560 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:39,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/a28ae47130bc4a5eb343bf9ccc9b9398 2024-11-20T16:54:39,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/0ed97973d0ca41daaf7d6d7161191a85 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/0ed97973d0ca41daaf7d6d7161191a85 2024-11-20T16:54:39,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/0ed97973d0ca41daaf7d6d7161191a85, entries=150, sequenceid=474, filesize=12.0 K 2024-11-20T16:54:39,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/ba5f236296dc4e72b2d2bdc874be7188 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/ba5f236296dc4e72b2d2bdc874be7188 2024-11-20T16:54:39,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/ba5f236296dc4e72b2d2bdc874be7188, entries=150, sequenceid=474, filesize=12.0 K 2024-11-20T16:54:39,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/a28ae47130bc4a5eb343bf9ccc9b9398 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/a28ae47130bc4a5eb343bf9ccc9b9398 2024-11-20T16:54:39,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/a28ae47130bc4a5eb343bf9ccc9b9398, entries=150, sequenceid=474, filesize=12.0 K 2024-11-20T16:54:39,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=13.42 KB/13740 for 81890a01db534dee41b8dda5131aae4e in 1267ms, sequenceid=474, compaction requested=true 2024-11-20T16:54:39,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:39,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:54:39,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:39,897 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:54:39,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:54:39,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:39,898 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:54:39,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81890a01db534dee41b8dda5131aae4e:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:54:39,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:39,899 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:54:39,899 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54800 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:54:39,899 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/B is initiating minor compaction (all files) 2024-11-20T16:54:39,899 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/A is initiating minor compaction (all files) 2024-11-20T16:54:39,899 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/B in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:39,899 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/A in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:39,899 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/24593330381c41ed992e8c1b5a258fc8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/2185ebfb4c624689bff7cfd9f6411074, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/f0a01a94c92a4a7eaed467982e60994e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/ba5f236296dc4e72b2d2bdc874be7188] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=48.8 K 2024-11-20T16:54:39,899 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/36e4139d51434ed4a884a27bcdbb1b7b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/b5648455d1c44b989b98f6273733d38d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/c9cdbae5aec54431954b3bdd1644e868, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/0ed97973d0ca41daaf7d6d7161191a85] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=53.5 K 2024-11-20T16:54:39,899 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 24593330381c41ed992e8c1b5a258fc8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732121673597 2024-11-20T16:54:39,899 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36e4139d51434ed4a884a27bcdbb1b7b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732121673597 2024-11-20T16:54:39,899 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 2185ebfb4c624689bff7cfd9f6411074, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732121673955 2024-11-20T16:54:39,899 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5648455d1c44b989b98f6273733d38d, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732121673950 2024-11-20T16:54:39,900 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9cdbae5aec54431954b3bdd1644e868, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732121675246 2024-11-20T16:54:39,900 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting f0a01a94c92a4a7eaed467982e60994e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732121675246 2024-11-20T16:54:39,900 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ed97973d0ca41daaf7d6d7161191a85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1732121676451 2024-11-20T16:54:39,900 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting ba5f236296dc4e72b2d2bdc874be7188, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1732121676451 2024-11-20T16:54:39,907 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#A#compaction#475 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:39,907 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#B#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:39,907 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/740b8d8695e545fc904fb7902510a02c is 50, key is test_row_0/A:col10/1732121676451/Put/seqid=0 2024-11-20T16:54:39,907 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/fe8f94f7bc754936b89efbd554943921 is 50, key is test_row_0/B:col10/1732121676451/Put/seqid=0 2024-11-20T16:54:39,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742387_1563 (size=13153) 2024-11-20T16:54:39,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742386_1562 (size=13153) 2024-11-20T16:54:40,319 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/fe8f94f7bc754936b89efbd554943921 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/fe8f94f7bc754936b89efbd554943921 2024-11-20T16:54:40,320 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/740b8d8695e545fc904fb7902510a02c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/740b8d8695e545fc904fb7902510a02c 2024-11-20T16:54:40,323 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/B of 81890a01db534dee41b8dda5131aae4e into fe8f94f7bc754936b89efbd554943921(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:40,323 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/A of 81890a01db534dee41b8dda5131aae4e into 740b8d8695e545fc904fb7902510a02c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:40,323 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:40,323 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:40,323 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/B, priority=12, startTime=1732121679897; duration=0sec 2024-11-20T16:54:40,323 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/A, priority=12, startTime=1732121679897; duration=0sec 2024-11-20T16:54:40,324 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:40,324 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:B 2024-11-20T16:54:40,324 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:40,324 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:A 2024-11-20T16:54:40,324 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:54:40,324 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:54:40,325 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 81890a01db534dee41b8dda5131aae4e/C is initiating minor compaction (all files) 2024-11-20T16:54:40,325 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81890a01db534dee41b8dda5131aae4e/C in TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:40,325 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/808bf5b28d094326a90f54fc7fc5d445, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/e9cab47644654cabb96d8d0a884a2ef3, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/405166604b6f4b139d997cba00cdf42a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/a28ae47130bc4a5eb343bf9ccc9b9398] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp, totalSize=48.8 K 2024-11-20T16:54:40,325 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 808bf5b28d094326a90f54fc7fc5d445, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732121673597 2024-11-20T16:54:40,325 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting e9cab47644654cabb96d8d0a884a2ef3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732121673955 2024-11-20T16:54:40,325 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 405166604b6f4b139d997cba00cdf42a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732121675246 2024-11-20T16:54:40,325 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting a28ae47130bc4a5eb343bf9ccc9b9398, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1732121676451 2024-11-20T16:54:40,332 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81890a01db534dee41b8dda5131aae4e#C#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:40,332 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/2fc757809dd4417eb4856a8ea7214275 is 50, key is test_row_0/C:col10/1732121676451/Put/seqid=0 2024-11-20T16:54:40,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742388_1564 (size=13153) 2024-11-20T16:54:40,743 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/2fc757809dd4417eb4856a8ea7214275 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/2fc757809dd4417eb4856a8ea7214275 2024-11-20T16:54:40,749 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81890a01db534dee41b8dda5131aae4e/C of 81890a01db534dee41b8dda5131aae4e into 2fc757809dd4417eb4856a8ea7214275(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:40,749 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81890a01db534dee41b8dda5131aae4e: 2024-11-20T16:54:40,750 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., storeName=81890a01db534dee41b8dda5131aae4e/C, priority=12, startTime=1732121679898; duration=0sec 2024-11-20T16:54:40,750 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:40,750 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81890a01db534dee41b8dda5131aae4e:C 2024-11-20T16:54:45,518 INFO [Thread-2156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2156. 2024-11-20T16:54:45,518 DEBUG [Thread-2156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:45,518 DEBUG [Thread-2156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:45,518 DEBUG [Thread-2156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:45,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:45,601 INFO [Thread-2162 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2162. 2024-11-20T16:54:45,601 DEBUG [Thread-2162 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:54:45,601 DEBUG [Thread-2162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:45,601 DEBUG [Thread-2162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:45,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:45,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T16:54:45,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-20T16:54:45,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 148 2024-11-20T16:54:45,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-20T16:54:45,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-20T16:54:45,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 167 2024-11-20T16:54:45,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T16:54:45,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T16:54:45,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1189 2024-11-20T16:54:45,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3567 rows 2024-11-20T16:54:45,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1186 2024-11-20T16:54:45,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3555 rows 2024-11-20T16:54:45,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1180 2024-11-20T16:54:45,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3540 rows 2024-11-20T16:54:45,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1178 2024-11-20T16:54:45,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3534 rows 2024-11-20T16:54:45,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1192 2024-11-20T16:54:45,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3571 rows 2024-11-20T16:54:45,604 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T16:54:45,604 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T16:54:45,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:45,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:45,605 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:54:45,606 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T16:54:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T16:54:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-20T16:54:45,609 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121685609"}]},"ts":"1732121685609"} 2024-11-20T16:54:45,610 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T16:54:45,610 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T16:54:45,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T16:54:45,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81890a01db534dee41b8dda5131aae4e, UNASSIGN}] 2024-11-20T16:54:45,613 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81890a01db534dee41b8dda5131aae4e, UNASSIGN 2024-11-20T16:54:45,613 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=81890a01db534dee41b8dda5131aae4e, regionState=CLOSING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:45,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81890a01db534dee41b8dda5131aae4e, UNASSIGN because future has completed 2024-11-20T16:54:45,615 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T16:54:45,615 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:54:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-20T16:54:45,768 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:45,768 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T16:54:45,768 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 81890a01db534dee41b8dda5131aae4e, disabling compactions & flushes 2024-11-20T16:54:45,768 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:45,769 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:45,769 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. after waiting 0 ms 2024-11-20T16:54:45,769 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:45,769 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(2902): Flushing 81890a01db534dee41b8dda5131aae4e 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-20T16:54:45,769 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=A 2024-11-20T16:54:45,769 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:45,770 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=B 2024-11-20T16:54:45,770 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:45,770 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 81890a01db534dee41b8dda5131aae4e, store=C 2024-11-20T16:54:45,770 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:45,777 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/187a74c7a1544834a700ce14a02c9c04 is 50, key is test_row_0/A:col10/1732121679557/Put/seqid=0 2024-11-20T16:54:45,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742389_1565 (size=12301) 2024-11-20T16:54:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-20T16:54:46,182 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/187a74c7a1544834a700ce14a02c9c04 2024-11-20T16:54:46,195 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/d6bd60102d0d44deb19ae03c1c7cec22 is 50, key is test_row_0/B:col10/1732121679557/Put/seqid=0 2024-11-20T16:54:46,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742390_1566 (size=12301) 2024-11-20T16:54:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-20T16:54:46,600 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/d6bd60102d0d44deb19ae03c1c7cec22 2024-11-20T16:54:46,615 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/aa30a5e5ad344a61bb91c664992dd018 is 50, key is test_row_0/C:col10/1732121679557/Put/seqid=0 2024-11-20T16:54:46,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742391_1567 (size=12301) 2024-11-20T16:54:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-20T16:54:47,019 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/aa30a5e5ad344a61bb91c664992dd018 2024-11-20T16:54:47,022 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/A/187a74c7a1544834a700ce14a02c9c04 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/187a74c7a1544834a700ce14a02c9c04 2024-11-20T16:54:47,025 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/187a74c7a1544834a700ce14a02c9c04, entries=150, sequenceid=484, filesize=12.0 K 2024-11-20T16:54:47,026 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/B/d6bd60102d0d44deb19ae03c1c7cec22 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/d6bd60102d0d44deb19ae03c1c7cec22 2024-11-20T16:54:47,029 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/d6bd60102d0d44deb19ae03c1c7cec22, entries=150, sequenceid=484, filesize=12.0 K 2024-11-20T16:54:47,029 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/.tmp/C/aa30a5e5ad344a61bb91c664992dd018 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/aa30a5e5ad344a61bb91c664992dd018 2024-11-20T16:54:47,033 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/aa30a5e5ad344a61bb91c664992dd018, entries=150, sequenceid=484, filesize=12.0 K 2024-11-20T16:54:47,033 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 81890a01db534dee41b8dda5131aae4e in 1264ms, sequenceid=484, compaction requested=false 2024-11-20T16:54:47,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/ae697bbced1f4ae6aadf534eaf2e1773, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/5d4722ff2f9049cbb2f77038f8eb9d9f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/ca8d834242ce42c48156df3dec5ae51d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/cdfc27b7975249aebc1d0df8033647d8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/907712d1d05447d1bd432637d3bd6b4d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/75c613c5a41d4a5f9b60bba813383be3, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/ccd137b969294ba2bccc6db21ea7de7e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/bd5d619808264156945ff1134beb5930, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/a94f72503db64c56a2cf0dee86901ddf, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/6efdd1b69084470a9e9c813024862606, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/badf0a45358647aaa08b3b849acf4714, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/2243e286cc1d444abfd9a620572181dc, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/831f0fccd12a45869feecd5478201fa1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/fe86cf5066304c92979695af3cbe60c4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/b7aa385651de487da2ef7f28473b6729, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/93b6a269daf24081957b2960ebe270b5, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/835475c8335e4c829a155c891db133fb, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/96bc30758bfd4b98bd898b91e9c2d50a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/9f59c3772a294012a5eb99b643e8c2af, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/4d31364fd7604a7ea538a26d3aae1a73, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/6d8ef5e28480437dafb13cccba60126c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/36e4139d51434ed4a884a27bcdbb1b7b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/b5648455d1c44b989b98f6273733d38d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/c9cdbae5aec54431954b3bdd1644e868, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/0ed97973d0ca41daaf7d6d7161191a85] to archive 2024-11-20T16:54:47,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T16:54:47,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/ae697bbced1f4ae6aadf534eaf2e1773 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/ae697bbced1f4ae6aadf534eaf2e1773 2024-11-20T16:54:47,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/5d4722ff2f9049cbb2f77038f8eb9d9f to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/5d4722ff2f9049cbb2f77038f8eb9d9f 2024-11-20T16:54:47,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/ca8d834242ce42c48156df3dec5ae51d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/ca8d834242ce42c48156df3dec5ae51d 2024-11-20T16:54:47,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/cdfc27b7975249aebc1d0df8033647d8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/cdfc27b7975249aebc1d0df8033647d8 2024-11-20T16:54:47,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/907712d1d05447d1bd432637d3bd6b4d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/907712d1d05447d1bd432637d3bd6b4d 2024-11-20T16:54:47,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/75c613c5a41d4a5f9b60bba813383be3 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/75c613c5a41d4a5f9b60bba813383be3 2024-11-20T16:54:47,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/ccd137b969294ba2bccc6db21ea7de7e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/ccd137b969294ba2bccc6db21ea7de7e 2024-11-20T16:54:47,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/bd5d619808264156945ff1134beb5930 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/bd5d619808264156945ff1134beb5930 2024-11-20T16:54:47,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/a94f72503db64c56a2cf0dee86901ddf to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/a94f72503db64c56a2cf0dee86901ddf 2024-11-20T16:54:47,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/6efdd1b69084470a9e9c813024862606 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/6efdd1b69084470a9e9c813024862606 2024-11-20T16:54:47,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/badf0a45358647aaa08b3b849acf4714 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/badf0a45358647aaa08b3b849acf4714 2024-11-20T16:54:47,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/2243e286cc1d444abfd9a620572181dc to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/2243e286cc1d444abfd9a620572181dc 2024-11-20T16:54:47,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/831f0fccd12a45869feecd5478201fa1 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/831f0fccd12a45869feecd5478201fa1 2024-11-20T16:54:47,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/fe86cf5066304c92979695af3cbe60c4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/fe86cf5066304c92979695af3cbe60c4 2024-11-20T16:54:47,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/b7aa385651de487da2ef7f28473b6729 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/b7aa385651de487da2ef7f28473b6729 2024-11-20T16:54:47,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/93b6a269daf24081957b2960ebe270b5 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/93b6a269daf24081957b2960ebe270b5 2024-11-20T16:54:47,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/835475c8335e4c829a155c891db133fb to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/835475c8335e4c829a155c891db133fb 2024-11-20T16:54:47,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/96bc30758bfd4b98bd898b91e9c2d50a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/96bc30758bfd4b98bd898b91e9c2d50a 2024-11-20T16:54:47,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/9f59c3772a294012a5eb99b643e8c2af to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/9f59c3772a294012a5eb99b643e8c2af 2024-11-20T16:54:47,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/4d31364fd7604a7ea538a26d3aae1a73 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/4d31364fd7604a7ea538a26d3aae1a73 2024-11-20T16:54:47,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/6d8ef5e28480437dafb13cccba60126c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/6d8ef5e28480437dafb13cccba60126c 2024-11-20T16:54:47,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/36e4139d51434ed4a884a27bcdbb1b7b to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/36e4139d51434ed4a884a27bcdbb1b7b 2024-11-20T16:54:47,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/b5648455d1c44b989b98f6273733d38d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/b5648455d1c44b989b98f6273733d38d 2024-11-20T16:54:47,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/c9cdbae5aec54431954b3bdd1644e868 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/c9cdbae5aec54431954b3bdd1644e868 2024-11-20T16:54:47,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/0ed97973d0ca41daaf7d6d7161191a85 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/0ed97973d0ca41daaf7d6d7161191a85 2024-11-20T16:54:47,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/20908969045c4ca485eac43f6efa29cb, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/3b4a2358a6c54ec9abf37088a42a1b13, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/6ce72fdf97844993be4321e03835365d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/7a70dd2c3e3a4cc4a66d723cfd6290c5, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/0fb829d766ee47929da90087d2937590, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/715fb96de16f463aa2e931716e95744a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/68565cd2e75a43bc97db025caef07725, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/4f2e6f73581a4250b0b390ef36f8dfc3, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/6b741fce5f2e4480843fd079bf1b8de6, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/1a2dc0af6da44336aaee7796f229988c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/80c8bf92ed314580a28488d92e6f8260, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/c0beb5e454e347f99ec0c2dba0fab33f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/37ee42b28841450a875c4e12bf46e6f6, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/a9954e75f83b4e58a957038453ae5aa0, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/09c58671a72c41d48dac29be71ee80eb, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/623734bdf1244b8884b953e621f6644a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/03c3e9d19aa8408e997574e5a043b51b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/1cf954f78fac4e09bf899676a88eae3f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/c9b409364b5646a6bda519b792714afa, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/b84534b22f7b452086c7e8424c6a966a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/24593330381c41ed992e8c1b5a258fc8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/2386e450cbad46f79b0e967c963f64b2, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/2185ebfb4c624689bff7cfd9f6411074, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/f0a01a94c92a4a7eaed467982e60994e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/ba5f236296dc4e72b2d2bdc874be7188] to archive 2024-11-20T16:54:47,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T16:54:47,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/20908969045c4ca485eac43f6efa29cb to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/20908969045c4ca485eac43f6efa29cb 2024-11-20T16:54:47,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/3b4a2358a6c54ec9abf37088a42a1b13 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/3b4a2358a6c54ec9abf37088a42a1b13 2024-11-20T16:54:47,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/6ce72fdf97844993be4321e03835365d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/6ce72fdf97844993be4321e03835365d 2024-11-20T16:54:47,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/7a70dd2c3e3a4cc4a66d723cfd6290c5 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/7a70dd2c3e3a4cc4a66d723cfd6290c5 2024-11-20T16:54:47,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/0fb829d766ee47929da90087d2937590 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/0fb829d766ee47929da90087d2937590 2024-11-20T16:54:47,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/715fb96de16f463aa2e931716e95744a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/715fb96de16f463aa2e931716e95744a 2024-11-20T16:54:47,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/68565cd2e75a43bc97db025caef07725 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/68565cd2e75a43bc97db025caef07725 2024-11-20T16:54:47,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/4f2e6f73581a4250b0b390ef36f8dfc3 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/4f2e6f73581a4250b0b390ef36f8dfc3 2024-11-20T16:54:47,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/6b741fce5f2e4480843fd079bf1b8de6 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/6b741fce5f2e4480843fd079bf1b8de6 2024-11-20T16:54:47,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/1a2dc0af6da44336aaee7796f229988c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/1a2dc0af6da44336aaee7796f229988c 2024-11-20T16:54:47,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/80c8bf92ed314580a28488d92e6f8260 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/80c8bf92ed314580a28488d92e6f8260 2024-11-20T16:54:47,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/c0beb5e454e347f99ec0c2dba0fab33f to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/c0beb5e454e347f99ec0c2dba0fab33f 2024-11-20T16:54:47,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/37ee42b28841450a875c4e12bf46e6f6 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/37ee42b28841450a875c4e12bf46e6f6 2024-11-20T16:54:47,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/a9954e75f83b4e58a957038453ae5aa0 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/a9954e75f83b4e58a957038453ae5aa0 2024-11-20T16:54:47,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/09c58671a72c41d48dac29be71ee80eb to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/09c58671a72c41d48dac29be71ee80eb 2024-11-20T16:54:47,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/623734bdf1244b8884b953e621f6644a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/623734bdf1244b8884b953e621f6644a 2024-11-20T16:54:47,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/03c3e9d19aa8408e997574e5a043b51b to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/03c3e9d19aa8408e997574e5a043b51b 2024-11-20T16:54:47,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/1cf954f78fac4e09bf899676a88eae3f to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/1cf954f78fac4e09bf899676a88eae3f 2024-11-20T16:54:47,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/c9b409364b5646a6bda519b792714afa to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/c9b409364b5646a6bda519b792714afa 2024-11-20T16:54:47,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/b84534b22f7b452086c7e8424c6a966a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/b84534b22f7b452086c7e8424c6a966a 2024-11-20T16:54:47,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/24593330381c41ed992e8c1b5a258fc8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/24593330381c41ed992e8c1b5a258fc8 2024-11-20T16:54:47,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/2386e450cbad46f79b0e967c963f64b2 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/2386e450cbad46f79b0e967c963f64b2 2024-11-20T16:54:47,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/2185ebfb4c624689bff7cfd9f6411074 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/2185ebfb4c624689bff7cfd9f6411074 2024-11-20T16:54:47,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/f0a01a94c92a4a7eaed467982e60994e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/f0a01a94c92a4a7eaed467982e60994e 2024-11-20T16:54:47,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/ba5f236296dc4e72b2d2bdc874be7188 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/ba5f236296dc4e72b2d2bdc874be7188 2024-11-20T16:54:47,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/3e2c4f3d5a104eb494d8c8ff6ca3e2cd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/14ed59d87ab34909bc64f194d97264dd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/d4c7a59ee0d143c992e56163dfab3de5, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/ee7b2222c19148468c4e4acc76e8bcea, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/25f6dc0288354621bcdc8092510dccaf, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/5ae5757d30314a519fc4f0610239dd69, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/849953e9b55245819fc2f0aef206ea0d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/6864a74719a04d45a819054e9341e4c4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/17adca6e4b494dbeadeded6ea348bf31, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/f1792067e0b848d1b027ff89ba3b2dd3, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/bd272765bf0d4ae58e17f71a5352e6b3, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/31fe1b5457c94ef686a2b2df59078cf0, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/2d6db689f4c540b7bfcdabfcbb6e0b6d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/5614931662694cf3bc9331376f96b057, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/221c51b330184dccaa1fb6e8a0ed7b96, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/d20c124054894289a8963ba5dda2c77f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/184bbd39ecdd489f9381bd77bee9876c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/1863135083bf490c9b5663a9a4f5a805, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/de1df42d82c142a695d476616ca793ad, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/15d67ec37b1c420cbaa1955c8bf43f3e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/808bf5b28d094326a90f54fc7fc5d445, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/e5ea2d80eb40427da6bd502565033e2f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/e9cab47644654cabb96d8d0a884a2ef3, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/405166604b6f4b139d997cba00cdf42a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/a28ae47130bc4a5eb343bf9ccc9b9398] to archive 2024-11-20T16:54:47,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T16:54:47,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/3e2c4f3d5a104eb494d8c8ff6ca3e2cd to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/3e2c4f3d5a104eb494d8c8ff6ca3e2cd 2024-11-20T16:54:47,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/14ed59d87ab34909bc64f194d97264dd to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/14ed59d87ab34909bc64f194d97264dd 2024-11-20T16:54:47,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/d4c7a59ee0d143c992e56163dfab3de5 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/d4c7a59ee0d143c992e56163dfab3de5 2024-11-20T16:54:47,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/ee7b2222c19148468c4e4acc76e8bcea to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/ee7b2222c19148468c4e4acc76e8bcea 2024-11-20T16:54:47,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/25f6dc0288354621bcdc8092510dccaf to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/25f6dc0288354621bcdc8092510dccaf 2024-11-20T16:54:47,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/5ae5757d30314a519fc4f0610239dd69 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/5ae5757d30314a519fc4f0610239dd69 2024-11-20T16:54:47,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/849953e9b55245819fc2f0aef206ea0d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/849953e9b55245819fc2f0aef206ea0d 2024-11-20T16:54:47,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/6864a74719a04d45a819054e9341e4c4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/6864a74719a04d45a819054e9341e4c4 2024-11-20T16:54:47,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/17adca6e4b494dbeadeded6ea348bf31 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/17adca6e4b494dbeadeded6ea348bf31 2024-11-20T16:54:47,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/f1792067e0b848d1b027ff89ba3b2dd3 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/f1792067e0b848d1b027ff89ba3b2dd3 2024-11-20T16:54:47,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/bd272765bf0d4ae58e17f71a5352e6b3 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/bd272765bf0d4ae58e17f71a5352e6b3 2024-11-20T16:54:47,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/31fe1b5457c94ef686a2b2df59078cf0 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/31fe1b5457c94ef686a2b2df59078cf0 2024-11-20T16:54:47,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/2d6db689f4c540b7bfcdabfcbb6e0b6d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/2d6db689f4c540b7bfcdabfcbb6e0b6d 2024-11-20T16:54:47,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/5614931662694cf3bc9331376f96b057 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/5614931662694cf3bc9331376f96b057 2024-11-20T16:54:47,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/221c51b330184dccaa1fb6e8a0ed7b96 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/221c51b330184dccaa1fb6e8a0ed7b96 2024-11-20T16:54:47,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/d20c124054894289a8963ba5dda2c77f to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/d20c124054894289a8963ba5dda2c77f 2024-11-20T16:54:47,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/184bbd39ecdd489f9381bd77bee9876c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/184bbd39ecdd489f9381bd77bee9876c 2024-11-20T16:54:47,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/1863135083bf490c9b5663a9a4f5a805 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/1863135083bf490c9b5663a9a4f5a805 2024-11-20T16:54:47,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/de1df42d82c142a695d476616ca793ad to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/de1df42d82c142a695d476616ca793ad 2024-11-20T16:54:47,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/15d67ec37b1c420cbaa1955c8bf43f3e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/15d67ec37b1c420cbaa1955c8bf43f3e 2024-11-20T16:54:47,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/808bf5b28d094326a90f54fc7fc5d445 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/808bf5b28d094326a90f54fc7fc5d445 2024-11-20T16:54:47,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/e5ea2d80eb40427da6bd502565033e2f to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/e5ea2d80eb40427da6bd502565033e2f 2024-11-20T16:54:47,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/e9cab47644654cabb96d8d0a884a2ef3 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/e9cab47644654cabb96d8d0a884a2ef3 2024-11-20T16:54:47,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/405166604b6f4b139d997cba00cdf42a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/405166604b6f4b139d997cba00cdf42a 2024-11-20T16:54:47,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/a28ae47130bc4a5eb343bf9ccc9b9398 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/a28ae47130bc4a5eb343bf9ccc9b9398 2024-11-20T16:54:47,125 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/recovered.edits/487.seqid, newMaxSeqId=487, maxSeqId=1 2024-11-20T16:54:47,126 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. 2024-11-20T16:54:47,126 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 81890a01db534dee41b8dda5131aae4e: Waiting for close lock at 1732121685768Running coprocessor pre-close hooks at 1732121685768Disabling compacts and flushes for region at 1732121685768Disabling writes for close at 1732121685769 (+1 ms)Obtaining lock to block concurrent updates at 1732121685769Preparing flush snapshotting stores in 81890a01db534dee41b8dda5131aae4e at 1732121685769Finished memstore snapshotting TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732121685770 (+1 ms)Flushing stores of TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e. at 1732121685771 (+1 ms)Flushing 81890a01db534dee41b8dda5131aae4e/A: creating writer at 1732121685771Flushing 81890a01db534dee41b8dda5131aae4e/A: appending metadata at 1732121685777 (+6 ms)Flushing 81890a01db534dee41b8dda5131aae4e/A: closing flushed file at 1732121685777Flushing 81890a01db534dee41b8dda5131aae4e/B: creating writer at 1732121686191 (+414 ms)Flushing 81890a01db534dee41b8dda5131aae4e/B: appending metadata at 1732121686194 (+3 ms)Flushing 81890a01db534dee41b8dda5131aae4e/B: closing flushed file at 1732121686194Flushing 81890a01db534dee41b8dda5131aae4e/C: creating writer at 1732121686609 (+415 ms)Flushing 81890a01db534dee41b8dda5131aae4e/C: appending metadata at 1732121686615 (+6 ms)Flushing 81890a01db534dee41b8dda5131aae4e/C: closing flushed file at 1732121686615Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65ed601b: reopening flushed file at 1732121687022 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cd97a7f: reopening flushed file at 1732121687025 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e2e00c7: reopening flushed file at 1732121687029 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 81890a01db534dee41b8dda5131aae4e in 1264ms, sequenceid=484, compaction requested=false at 1732121687033 (+4 ms)Writing region close event to WAL at 1732121687122 (+89 ms)Running coprocessor post-close hooks at 1732121687126 (+4 ms)Closed at 1732121687126 2024-11-20T16:54:47,128 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:47,128 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=81890a01db534dee41b8dda5131aae4e, regionState=CLOSED 2024-11-20T16:54:47,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 because future has completed 2024-11-20T16:54:47,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-20T16:54:47,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 81890a01db534dee41b8dda5131aae4e, server=b60eac9bda43,33929,1732121526384 in 1.5150 sec 2024-11-20T16:54:47,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-20T16:54:47,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81890a01db534dee41b8dda5131aae4e, UNASSIGN in 1.5200 sec 2024-11-20T16:54:47,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-20T16:54:47,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5220 sec 2024-11-20T16:54:47,136 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121687136"}]},"ts":"1732121687136"} 2024-11-20T16:54:47,137 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T16:54:47,137 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T16:54:47,139 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5320 sec 2024-11-20T16:54:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-20T16:54:47,767 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:47,768 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T16:54:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:54:47,772 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:54:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-20T16:54:47,774 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=151, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:54:47,779 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:47,783 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A, FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B, FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C, FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/recovered.edits] 2024-11-20T16:54:47,787 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/187a74c7a1544834a700ce14a02c9c04 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/187a74c7a1544834a700ce14a02c9c04 2024-11-20T16:54:47,788 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/740b8d8695e545fc904fb7902510a02c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/A/740b8d8695e545fc904fb7902510a02c 2024-11-20T16:54:47,791 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/d6bd60102d0d44deb19ae03c1c7cec22 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/d6bd60102d0d44deb19ae03c1c7cec22 2024-11-20T16:54:47,793 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/fe8f94f7bc754936b89efbd554943921 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/B/fe8f94f7bc754936b89efbd554943921 2024-11-20T16:54:47,795 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/2fc757809dd4417eb4856a8ea7214275 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/2fc757809dd4417eb4856a8ea7214275 2024-11-20T16:54:47,797 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/aa30a5e5ad344a61bb91c664992dd018 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/C/aa30a5e5ad344a61bb91c664992dd018 2024-11-20T16:54:47,800 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/recovered.edits/487.seqid to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e/recovered.edits/487.seqid 2024-11-20T16:54:47,801 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/81890a01db534dee41b8dda5131aae4e 2024-11-20T16:54:47,801 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T16:54:47,804 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=151, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:54:47,807 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T16:54:47,810 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T16:54:47,811 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=151, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:54:47,811 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T16:54:47,812 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732121687812"}]},"ts":"9223372036854775807"} 2024-11-20T16:54:47,820 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T16:54:47,820 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 81890a01db534dee41b8dda5131aae4e, NAME => 'TestAcidGuarantees,,1732121654350.81890a01db534dee41b8dda5131aae4e.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T16:54:47,820 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T16:54:47,820 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732121687820"}]},"ts":"9223372036854775807"} 2024-11-20T16:54:47,822 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T16:54:47,822 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=151, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:54:47,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 54 msec 2024-11-20T16:54:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-20T16:54:47,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T16:54:47,886 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:47,893 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247 (was 248), OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=462 (was 603), ProcessCount=11 (was 11), AvailableMemoryMB=7266 (was 7489) 2024-11-20T16:54:47,901 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=462, ProcessCount=11, AvailableMemoryMB=7265 2024-11-20T16:54:47,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T16:54:47,903 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T16:54:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T16:54:47,905 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T16:54:47,905 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:47,905 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 152 2024-11-20T16:54:47,906 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T16:54:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T16:54:47,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742392_1568 (size=1061) 2024-11-20T16:54:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T16:54:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T16:54:48,316 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625 2024-11-20T16:54:48,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742393_1569 (size=53) 2024-11-20T16:54:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T16:54:48,727 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:54:48,727 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7565e0cf39c553ea85172f8e95032446, disabling compactions & flushes 2024-11-20T16:54:48,727 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:48,727 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:48,727 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. after waiting 0 ms 2024-11-20T16:54:48,727 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:48,727 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:48,728 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7565e0cf39c553ea85172f8e95032446: Waiting for close lock at 1732121688727Disabling compacts and flushes for region at 1732121688727Disabling writes for close at 1732121688727Writing region close event to WAL at 1732121688727Closed at 1732121688727 2024-11-20T16:54:48,730 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T16:54:48,730 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732121688730"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732121688730"}]},"ts":"1732121688730"} 2024-11-20T16:54:48,734 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T16:54:48,736 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T16:54:48,737 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121688736"}]},"ts":"1732121688736"} 2024-11-20T16:54:48,738 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T16:54:48,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7565e0cf39c553ea85172f8e95032446, ASSIGN}] 2024-11-20T16:54:48,740 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7565e0cf39c553ea85172f8e95032446, ASSIGN 2024-11-20T16:54:48,741 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7565e0cf39c553ea85172f8e95032446, ASSIGN; state=OFFLINE, location=b60eac9bda43,33929,1732121526384; forceNewPlan=false, retain=false 2024-11-20T16:54:48,892 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=7565e0cf39c553ea85172f8e95032446, regionState=OPENING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:48,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7565e0cf39c553ea85172f8e95032446, ASSIGN because future has completed 2024-11-20T16:54:48,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:54:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T16:54:49,055 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:49,055 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7752): Opening region: {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} 2024-11-20T16:54:49,055 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:49,055 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:54:49,056 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7794): checking encryption for 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:49,056 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7797): checking classloading for 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:49,057 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:49,058 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:54:49,058 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7565e0cf39c553ea85172f8e95032446 columnFamilyName A 2024-11-20T16:54:49,059 DEBUG [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:49,059 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] regionserver.HStore(327): Store=7565e0cf39c553ea85172f8e95032446/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:54:49,059 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:49,060 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:54:49,061 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7565e0cf39c553ea85172f8e95032446 columnFamilyName B 2024-11-20T16:54:49,061 DEBUG [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:49,061 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] regionserver.HStore(327): Store=7565e0cf39c553ea85172f8e95032446/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:54:49,061 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:49,063 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:54:49,063 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7565e0cf39c553ea85172f8e95032446 columnFamilyName C 2024-11-20T16:54:49,063 DEBUG [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:49,063 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] regionserver.HStore(327): Store=7565e0cf39c553ea85172f8e95032446/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:54:49,064 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:49,064 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1038): replaying wal for 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:49,065 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:49,065 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:49,067 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1048): stopping wal replay for 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:49,067 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1060): Cleaning up temporary data for 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:49,067 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T16:54:49,069 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1093): writing seq id for 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:49,071 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T16:54:49,071 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1114): Opened 7565e0cf39c553ea85172f8e95032446; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72529952, jitterRate=0.0807805061340332}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T16:54:49,071 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:49,072 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1006): Region open journal for 7565e0cf39c553ea85172f8e95032446: Running coprocessor pre-open hook at 1732121689056Writing region info on filesystem at 1732121689056Initializing all the Stores at 1732121689056Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121689056Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121689057 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121689057Cleaning up temporary data from old regions at 1732121689067 (+10 ms)Running coprocessor post-open hooks at 1732121689071 (+4 ms)Region opened successfully at 1732121689072 (+1 ms) 2024-11-20T16:54:49,073 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., pid=154, masterSystemTime=1732121689051 2024-11-20T16:54:49,075 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:49,075 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:49,075 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=7565e0cf39c553ea85172f8e95032446, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:49,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 because future has completed 2024-11-20T16:54:49,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-20T16:54:49,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure 7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 in 181 msec 2024-11-20T16:54:49,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-20T16:54:49,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7565e0cf39c553ea85172f8e95032446, ASSIGN in 341 msec 2024-11-20T16:54:49,083 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T16:54:49,083 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121689083"}]},"ts":"1732121689083"} 2024-11-20T16:54:49,084 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T16:54:49,085 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T16:54:49,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-11-20T16:54:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T16:54:50,057 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:50,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66320673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:50,058 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:50,058 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:50,059 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:50,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:50,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:50,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7ef3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:50,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:50,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:50,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:50,060 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@224452e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:50,061 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:50,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:50,063 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T16:54:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is b60eac9bda43,33377,1732121525625 2024-11-20T16:54:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@98b75a5 2024-11-20T16:54:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T16:54:50,065 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T16:54:50,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T16:54:50,066 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T16:54:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=155, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T16:54:50,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742394_1570 (size=1097) 2024-11-20T16:54:50,479 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-20T16:54:50,479 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-20T16:54:50,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T16:54:50,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7565e0cf39c553ea85172f8e95032446, REOPEN/MOVE}] 2024-11-20T16:54:50,488 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7565e0cf39c553ea85172f8e95032446, REOPEN/MOVE 2024-11-20T16:54:50,489 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=7565e0cf39c553ea85172f8e95032446, regionState=CLOSING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:50,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7565e0cf39c553ea85172f8e95032446, REOPEN/MOVE because future has completed 2024-11-20T16:54:50,491 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T16:54:50,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:54:50,644 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(122): Close 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:50,644 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T16:54:50,644 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1722): Closing 7565e0cf39c553ea85172f8e95032446, disabling compactions & flushes 2024-11-20T16:54:50,644 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:50,644 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:50,645 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. after waiting 0 ms 2024-11-20T16:54:50,645 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:50,650 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T16:54:50,651 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:50,652 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1676): Region close journal for 7565e0cf39c553ea85172f8e95032446: Waiting for close lock at 1732121690644Running coprocessor pre-close hooks at 1732121690644Disabling compacts and flushes for region at 1732121690644Disabling writes for close at 1732121690645 (+1 ms)Writing region close event to WAL at 1732121690645Running coprocessor post-close hooks at 1732121690651 (+6 ms)Closed at 1732121690651 2024-11-20T16:54:50,652 WARN [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegionServer(3299): Not adding moved region record: 7565e0cf39c553ea85172f8e95032446 to self. 2024-11-20T16:54:50,654 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(157): Closed 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:50,655 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=7565e0cf39c553ea85172f8e95032446, regionState=CLOSED 2024-11-20T16:54:50,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 because future has completed 2024-11-20T16:54:50,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-20T16:54:50,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; CloseRegionProcedure 7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 in 167 msec 2024-11-20T16:54:50,662 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7565e0cf39c553ea85172f8e95032446, REOPEN/MOVE; state=CLOSED, location=b60eac9bda43,33929,1732121526384; forceNewPlan=false, retain=true 2024-11-20T16:54:50,813 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=7565e0cf39c553ea85172f8e95032446, regionState=OPENING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:50,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7565e0cf39c553ea85172f8e95032446, REOPEN/MOVE because future has completed 2024-11-20T16:54:50,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:54:50,977 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:50,977 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7752): Opening region: {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} 2024-11-20T16:54:50,978 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:50,978 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T16:54:50,979 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7794): checking encryption for 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:50,979 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7797): checking classloading for 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:50,982 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:50,982 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:54:50,983 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7565e0cf39c553ea85172f8e95032446 columnFamilyName A 2024-11-20T16:54:50,986 DEBUG [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:50,987 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] regionserver.HStore(327): Store=7565e0cf39c553ea85172f8e95032446/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:54:50,987 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:50,987 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:54:50,988 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7565e0cf39c553ea85172f8e95032446 columnFamilyName B 2024-11-20T16:54:50,988 DEBUG [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:50,988 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] regionserver.HStore(327): Store=7565e0cf39c553ea85172f8e95032446/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:54:50,988 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:50,988 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T16:54:50,988 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7565e0cf39c553ea85172f8e95032446 columnFamilyName C 2024-11-20T16:54:50,988 DEBUG [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:50,989 INFO [StoreOpener-7565e0cf39c553ea85172f8e95032446-1 {}] regionserver.HStore(327): Store=7565e0cf39c553ea85172f8e95032446/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T16:54:50,989 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:50,989 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1038): replaying wal for 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:50,990 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:50,991 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:50,991 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1048): stopping wal replay for 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:50,992 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1060): Cleaning up temporary data for 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:50,992 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T16:54:50,993 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1093): writing seq id for 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:50,994 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1114): Opened 7565e0cf39c553ea85172f8e95032446; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68764666, jitterRate=0.024673372507095337}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T16:54:50,994 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:50,994 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1006): Region open journal for 7565e0cf39c553ea85172f8e95032446: Running coprocessor pre-open hook at 1732121690979Writing region info on filesystem at 1732121690979Initializing all the Stores at 1732121690980 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121690981 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121690981Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732121690981Cleaning up temporary data from old regions at 1732121690992 (+11 ms)Running coprocessor post-open hooks at 1732121690994 (+2 ms)Region opened successfully at 1732121690994 2024-11-20T16:54:50,995 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., pid=159, masterSystemTime=1732121690969 2024-11-20T16:54:50,996 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:50,996 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:50,997 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=7565e0cf39c553ea85172f8e95032446, regionState=OPEN, openSeqNum=5, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:50,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 because future has completed 2024-11-20T16:54:51,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=157 2024-11-20T16:54:51,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=157, state=SUCCESS, hasLock=false; OpenRegionProcedure 7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 in 183 msec 2024-11-20T16:54:51,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-20T16:54:51,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7565e0cf39c553ea85172f8e95032446, REOPEN/MOVE in 514 msec 2024-11-20T16:54:51,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-20T16:54:51,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 520 msec 2024-11-20T16:54:51,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 938 msec 2024-11-20T16:54:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-11-20T16:54:51,005 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:51,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11971b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:51,006 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:51,007 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:51,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:51,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:51,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eae7abe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:51,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:51,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:51,008 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:51,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e4c5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,009 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:51,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:51,009 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:51,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:51,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:51,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@241f1e7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:51,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:51,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:51,010 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:51,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18e3f014, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,011 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:51,011 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:51,012 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c8ce0a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:51,012 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:51,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38c48a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,013 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:51,013 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:51,014 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:51,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:51,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:51,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ec056d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:51,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:51,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:51,015 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:51,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@776de51d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,016 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:51,016 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:51,016 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:51,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:51,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:51,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@217045c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:51,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:51,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:51,017 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77b9384, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:51,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:51,019 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:51,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:51,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:51,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@226591a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:51,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:51,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:51,020 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:51,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1449128, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,021 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,021 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:51,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:51,022 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366e6f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:51,023 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c419e56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,024 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:51,024 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:51,025 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67dbd564, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:51,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:51,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:51,026 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce0f495, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:51,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:51,028 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692df14d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:51,029 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:51,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b0a33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,030 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,031 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,33377,-1 for getting cluster id 2024-11-20T16:54:51,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T16:54:51,031 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e' 2024-11-20T16:54:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T16:54:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84fda3d2-fd8d-4c67-bdd9-2c85ca41c37e" 2024-11-20T16:54:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73292552, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,33377,-1] 2024-11-20T16:54:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T16:54:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:54:51,032 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T16:54:51,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78f6ecf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T16:54:51,033 DEBUG [Thread-2541 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:51,033 DEBUG [Thread-2547 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:51,034 DEBUG [Thread-2545 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:51,034 DEBUG [Thread-2539 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:51,035 DEBUG [Thread-2530 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:51,035 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:54:51,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:51,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:51,035 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:51,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:51,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-20T16:54:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:51,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:51,037 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:54:51,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:54:51,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:54:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T16:54:51,038 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:54:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:51,039 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:54:51,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:54:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:54:51,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:54:51,047 DEBUG [Thread-2536 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:51,048 DEBUG [Thread-2532 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:51,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:51,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:54:51,053 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:54:51,055 DEBUG [Thread-2528 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:51,056 DEBUG [Thread-2534 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:51,056 DEBUG [Thread-2528 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:51,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7565e0cf39c553ea85172f8e95032446 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T16:54:51,056 DEBUG [Thread-2534 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:51,056 DEBUG [Thread-2528 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:51,057 DEBUG [Thread-2534 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:51,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:51,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:51,059 DEBUG [Thread-2543 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T16:54:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:54:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:54:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,33929,1732121526384, seqNum=-1] 2024-11-20T16:54:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T16:54:51,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T16:54:51,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5] 2024-11-20T16:54:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=A 2024-11-20T16:54:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=B 2024-11-20T16:54:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=C 2024-11-20T16:54:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:51,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112000abf9e144484219ace637fb6d108ee1_7565e0cf39c553ea85172f8e95032446 is 50, key is test_row_0/A:col10/1732121691055/Put/seqid=0 2024-11-20T16:54:51,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732121691078, max timestamp=1732121691084Num uniques 859; 2024-11-20T16:54:51,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732121691078, max timestamp=1732121691084Num uniques 859; 2024-11-20T16:54:51,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:51,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:51,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121691078, max timestamp=1732121691084Num uniques 859; 2024-11-20T16:54:51,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:51,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742395_1571 (size=12154) 2024-11-20T16:54:51,097 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:51,100 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112000abf9e144484219ace637fb6d108ee1_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112000abf9e144484219ace637fb6d108ee1_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:51,100 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/f0bf7017aa554852b7f731d79bc13b18, store: [table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:54:51,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/f0bf7017aa554852b7f731d79bc13b18 is 175, key is test_row_0/A:col10/1732121691055/Put/seqid=0 2024-11-20T16:54:51,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742396_1572 (size=30955) 2024-11-20T16:54:51,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33980 deadline: 1732121751104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,110 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/f0bf7017aa554852b7f731d79bc13b18 2024-11-20T16:54:51,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33970 deadline: 1732121751108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33984 deadline: 1732121751109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121751109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121751110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/dffe009e180547a2a66f63c5e5ce8dde is 50, key is test_row_0/B:col10/1732121691055/Put/seqid=0 2024-11-20T16:54:51,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742397_1573 (size=12001) 2024-11-20T16:54:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T16:54:51,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T16:54:51,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:51,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:51,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:51,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:51,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:51,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33980 deadline: 1732121751216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33970 deadline: 1732121751216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33984 deadline: 1732121751226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121751226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121751226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T16:54:51,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:51,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:51,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:51,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:51,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T16:54:51,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33980 deadline: 1732121751426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33970 deadline: 1732121751426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33984 deadline: 1732121751436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121751436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121751436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T16:54:51,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:51,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:51,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:51,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:51,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:51,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/dffe009e180547a2a66f63c5e5ce8dde 2024-11-20T16:54:51,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/0d10e202e7ea419e86b5071d3eb28da4 is 50, key is test_row_0/C:col10/1732121691055/Put/seqid=0 2024-11-20T16:54:51,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742398_1574 (size=12001) 2024-11-20T16:54:51,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T16:54:51,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:51,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:51,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:51,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:51,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T16:54:51,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33980 deadline: 1732121751736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33970 deadline: 1732121751736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33984 deadline: 1732121751746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121751746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121751746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:51,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:51,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:51,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T16:54:51,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:51,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:51,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:51,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:51,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:51,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/0d10e202e7ea419e86b5071d3eb28da4 2024-11-20T16:54:51,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/f0bf7017aa554852b7f731d79bc13b18 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/f0bf7017aa554852b7f731d79bc13b18 2024-11-20T16:54:51,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/f0bf7017aa554852b7f731d79bc13b18, entries=150, sequenceid=15, filesize=30.2 K 2024-11-20T16:54:51,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/dffe009e180547a2a66f63c5e5ce8dde as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/dffe009e180547a2a66f63c5e5ce8dde 2024-11-20T16:54:51,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/dffe009e180547a2a66f63c5e5ce8dde, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T16:54:51,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/0d10e202e7ea419e86b5071d3eb28da4 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/0d10e202e7ea419e86b5071d3eb28da4 2024-11-20T16:54:51,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/0d10e202e7ea419e86b5071d3eb28da4, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T16:54:51,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 7565e0cf39c553ea85172f8e95032446 in 913ms, sequenceid=15, compaction requested=false 2024-11-20T16:54:51,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:54:51,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T16:54:51,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:51,983 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 7565e0cf39c553ea85172f8e95032446 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-20T16:54:51,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=A 2024-11-20T16:54:51,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:51,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=B 2024-11-20T16:54:51,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:51,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=C 2024-11-20T16:54:51,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:51,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120416afed65cb14468b67b6789c3602faf_7565e0cf39c553ea85172f8e95032446 is 50, key is test_row_0/A:col10/1732121691102/Put/seqid=0 2024-11-20T16:54:51,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742399_1575 (size=12154) 2024-11-20T16:54:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T16:54:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:52,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:52,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121752256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:52,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121752256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:52,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33980 deadline: 1732121752260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:52,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:52,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33970 deadline: 1732121752260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:52,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:52,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33984 deadline: 1732121752261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:52,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33980 deadline: 1732121752366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:52,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:52,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33970 deadline: 1732121752366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:52,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:52,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:52,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33984 deadline: 1732121752376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:52,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:52,398 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120416afed65cb14468b67b6789c3602faf_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120416afed65cb14468b67b6789c3602faf_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:52,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/2c65775d4f7c4d8e93d09621c2401562, store: [table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:54:52,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/2c65775d4f7c4d8e93d09621c2401562 is 175, key is test_row_0/A:col10/1732121691102/Put/seqid=0 2024-11-20T16:54:52,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742400_1576 (size=30955) 2024-11-20T16:54:52,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33980 deadline: 1732121752576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:52,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33970 deadline: 1732121752576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:52,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33984 deadline: 1732121752586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:52,643 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T16:54:52,803 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/2c65775d4f7c4d8e93d09621c2401562 2024-11-20T16:54:52,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/4a2dfa99b0804ff89101673641f4af7c is 50, key is test_row_0/B:col10/1732121691102/Put/seqid=0 2024-11-20T16:54:52,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742401_1577 (size=12001) 2024-11-20T16:54:52,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33980 deadline: 1732121752886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:52,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:52,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33970 deadline: 1732121752886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:52,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:52,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33984 deadline: 1732121752896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T16:54:53,213 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/4a2dfa99b0804ff89101673641f4af7c 2024-11-20T16:54:53,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/4f1c6dbe8e0b4e35a8585a454d784d79 is 50, key is test_row_0/C:col10/1732121691102/Put/seqid=0 2024-11-20T16:54:53,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742402_1578 (size=12001) 2024-11-20T16:54:53,222 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/4f1c6dbe8e0b4e35a8585a454d784d79 2024-11-20T16:54:53,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/2c65775d4f7c4d8e93d09621c2401562 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/2c65775d4f7c4d8e93d09621c2401562 2024-11-20T16:54:53,228 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/2c65775d4f7c4d8e93d09621c2401562, entries=150, sequenceid=51, filesize=30.2 K 2024-11-20T16:54:53,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/4a2dfa99b0804ff89101673641f4af7c as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/4a2dfa99b0804ff89101673641f4af7c 2024-11-20T16:54:53,231 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/4a2dfa99b0804ff89101673641f4af7c, entries=150, sequenceid=51, filesize=11.7 K 2024-11-20T16:54:53,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/4f1c6dbe8e0b4e35a8585a454d784d79 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/4f1c6dbe8e0b4e35a8585a454d784d79 2024-11-20T16:54:53,234 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/4f1c6dbe8e0b4e35a8585a454d784d79, entries=150, sequenceid=51, filesize=11.7 K 2024-11-20T16:54:53,235 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 7565e0cf39c553ea85172f8e95032446 in 1252ms, sequenceid=51, compaction requested=false 2024-11-20T16:54:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:54:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-20T16:54:53,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-20T16:54:53,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-20T16:54:53,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1990 sec 2024-11-20T16:54:53,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.2060 sec 2024-11-20T16:54:53,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:53,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7565e0cf39c553ea85172f8e95032446 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T16:54:53,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=A 2024-11-20T16:54:53,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:53,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=B 2024-11-20T16:54:53,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:53,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=C 2024-11-20T16:54:53,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:53,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209660b6e36c734243a7dbc61672f2633e_7565e0cf39c553ea85172f8e95032446 is 50, key is test_row_0/A:col10/1732121692260/Put/seqid=0 2024-11-20T16:54:53,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742403_1579 (size=14594) 2024-11-20T16:54:53,277 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:53,279 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209660b6e36c734243a7dbc61672f2633e_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209660b6e36c734243a7dbc61672f2633e_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:53,280 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/37b05625a689466f832dc1e3dbdb1688, store: [table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:54:53,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/37b05625a689466f832dc1e3dbdb1688 is 175, key is test_row_0/A:col10/1732121692260/Put/seqid=0 2024-11-20T16:54:53,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742404_1580 (size=39549) 2024-11-20T16:54:53,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121693296, max timestamp=1732121693308Num uniques 859; 2024-11-20T16:54:53,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:54:53,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121693296, max timestamp=1732121693308Num uniques 859; 2024-11-20T16:54:53,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:54:53,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121693296, max timestamp=1732121693308Num uniques 859; 2024-11-20T16:54:53,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:54:53,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:53,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121753353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121753353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:53,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:53,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:53,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33980 deadline: 1732121753396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:53,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33970 deadline: 1732121753396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:53,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:53,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33984 deadline: 1732121753406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:53,409 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:53,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:53,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121753466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:53,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121753466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:53,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121753676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:53,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121753676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:53,683 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/37b05625a689466f832dc1e3dbdb1688 2024-11-20T16:54:53,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/3ed74b5b571a459f9926e249b4c77585 is 50, key is test_row_0/B:col10/1732121692260/Put/seqid=0 2024-11-20T16:54:53,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742405_1581 (size=12001) 2024-11-20T16:54:53,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121753986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:53,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:53,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121753986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:53,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:53,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:53,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:54,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/3ed74b5b571a459f9926e249b4c77585 2024-11-20T16:54:54,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/1297a6cfe7bd4261a4eccc8067e02ff1 is 50, key is test_row_0/C:col10/1732121692260/Put/seqid=0 2024-11-20T16:54:54,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742406_1582 (size=12001) 2024-11-20T16:54:54,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33980 deadline: 1732121754406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:54,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33970 deadline: 1732121754406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:54,409 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 7565e0cf39c553ea85172f8e95032446 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:54,409 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 7565e0cf39c553ea85172f8e95032446 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:54,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33984 deadline: 1732121754416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:54,418 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 7565e0cf39c553ea85172f8e95032446 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:54,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121754496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:54,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121754496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:54,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/1297a6cfe7bd4261a4eccc8067e02ff1 2024-11-20T16:54:54,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/37b05625a689466f832dc1e3dbdb1688 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/37b05625a689466f832dc1e3dbdb1688 2024-11-20T16:54:54,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/37b05625a689466f832dc1e3dbdb1688, entries=200, sequenceid=62, filesize=38.6 K 2024-11-20T16:54:54,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/3ed74b5b571a459f9926e249b4c77585 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/3ed74b5b571a459f9926e249b4c77585 2024-11-20T16:54:54,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/3ed74b5b571a459f9926e249b4c77585, entries=150, sequenceid=62, filesize=11.7 K 2024-11-20T16:54:54,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/1297a6cfe7bd4261a4eccc8067e02ff1 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/1297a6cfe7bd4261a4eccc8067e02ff1 2024-11-20T16:54:54,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/1297a6cfe7bd4261a4eccc8067e02ff1, entries=150, sequenceid=62, filesize=11.7 K 2024-11-20T16:54:54,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 7565e0cf39c553ea85172f8e95032446 in 1247ms, sequenceid=62, compaction requested=true 2024-11-20T16:54:54,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:54:54,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:54:54,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:54,515 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:54:54,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:54:54,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:54,515 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:54:54,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:54:54,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:54,515 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:54:54,515 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:54:54,515 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/A is initiating minor compaction (all files) 2024-11-20T16:54:54,515 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/B is initiating minor compaction (all files) 2024-11-20T16:54:54,515 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/B in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:54,515 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/A in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:54,516 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/dffe009e180547a2a66f63c5e5ce8dde, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/4a2dfa99b0804ff89101673641f4af7c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/3ed74b5b571a459f9926e249b4c77585] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=35.2 K 2024-11-20T16:54:54,516 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/f0bf7017aa554852b7f731d79bc13b18, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/2c65775d4f7c4d8e93d09621c2401562, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/37b05625a689466f832dc1e3dbdb1688] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=99.1 K 2024-11-20T16:54:54,516 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:54,516 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/f0bf7017aa554852b7f731d79bc13b18, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/2c65775d4f7c4d8e93d09621c2401562, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/37b05625a689466f832dc1e3dbdb1688] 2024-11-20T16:54:54,516 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting dffe009e180547a2a66f63c5e5ce8dde, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732121691051 2024-11-20T16:54:54,516 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0bf7017aa554852b7f731d79bc13b18, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732121691051 2024-11-20T16:54:54,516 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a2dfa99b0804ff89101673641f4af7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732121691102 2024-11-20T16:54:54,516 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c65775d4f7c4d8e93d09621c2401562, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732121691102 2024-11-20T16:54:54,516 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ed74b5b571a459f9926e249b4c77585, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732121692259 2024-11-20T16:54:54,517 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37b05625a689466f832dc1e3dbdb1688, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732121692253 2024-11-20T16:54:54,520 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:54:54,522 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#B#compaction#490 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:54,522 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/072c727a69964d6085cf40ca2233fb16 is 50, key is test_row_0/B:col10/1732121692260/Put/seqid=0 2024-11-20T16:54:54,524 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120d629b3646c464ce798528cb12378fa29_7565e0cf39c553ea85172f8e95032446 store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:54:54,526 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120d629b3646c464ce798528cb12378fa29_7565e0cf39c553ea85172f8e95032446, store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:54:54,526 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d629b3646c464ce798528cb12378fa29_7565e0cf39c553ea85172f8e95032446 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:54:54,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742407_1583 (size=12104) 2024-11-20T16:54:54,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742408_1584 (size=4469) 2024-11-20T16:54:54,932 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/072c727a69964d6085cf40ca2233fb16 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/072c727a69964d6085cf40ca2233fb16 2024-11-20T16:54:54,932 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#A#compaction#491 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:54,933 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/0e15d72d64bb4d8b80bd6ff7c74c766e is 175, key is test_row_0/A:col10/1732121692260/Put/seqid=0 2024-11-20T16:54:54,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742409_1585 (size=31058) 2024-11-20T16:54:54,936 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/B of 7565e0cf39c553ea85172f8e95032446 into 072c727a69964d6085cf40ca2233fb16(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:54,936 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:54:54,936 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/B, priority=13, startTime=1732121694515; duration=0sec 2024-11-20T16:54:54,936 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:54,936 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:B 2024-11-20T16:54:54,937 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:54:54,937 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:54:54,937 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/C is initiating minor compaction (all files) 2024-11-20T16:54:54,937 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/C in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:54,937 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/0d10e202e7ea419e86b5071d3eb28da4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/4f1c6dbe8e0b4e35a8585a454d784d79, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/1297a6cfe7bd4261a4eccc8067e02ff1] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=35.2 K 2024-11-20T16:54:54,937 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d10e202e7ea419e86b5071d3eb28da4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732121691051 2024-11-20T16:54:54,938 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f1c6dbe8e0b4e35a8585a454d784d79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732121691102 2024-11-20T16:54:54,938 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 1297a6cfe7bd4261a4eccc8067e02ff1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732121692259 2024-11-20T16:54:54,939 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/0e15d72d64bb4d8b80bd6ff7c74c766e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/0e15d72d64bb4d8b80bd6ff7c74c766e 2024-11-20T16:54:54,942 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/A of 7565e0cf39c553ea85172f8e95032446 into 0e15d72d64bb4d8b80bd6ff7c74c766e(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:54,942 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:54:54,942 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/A, priority=13, startTime=1732121694515; duration=0sec 2024-11-20T16:54:54,943 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:54,943 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:A 2024-11-20T16:54:54,943 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#C#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:54,944 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/56c666c554a746c19b499e29131ebf34 is 50, key is test_row_0/C:col10/1732121692260/Put/seqid=0 2024-11-20T16:54:54,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742410_1586 (size=12104) 2024-11-20T16:54:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T16:54:55,206 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:55,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:54:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-20T16:54:55,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T16:54:55,208 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:54:55,209 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:54:55,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:54:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T16:54:55,350 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/56c666c554a746c19b499e29131ebf34 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/56c666c554a746c19b499e29131ebf34 2024-11-20T16:54:55,354 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/C of 7565e0cf39c553ea85172f8e95032446 into 56c666c554a746c19b499e29131ebf34(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:55,354 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:54:55,354 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/C, priority=13, startTime=1732121694515; duration=0sec 2024-11-20T16:54:55,354 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:55,354 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:C 2024-11-20T16:54:55,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T16:54:55,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:55,361 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 7565e0cf39c553ea85172f8e95032446 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-20T16:54:55,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=A 2024-11-20T16:54:55,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:55,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=B 2024-11-20T16:54:55,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:55,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=C 2024-11-20T16:54:55,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:55,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120af6be5fbc4c34993b1cbe2e849b7626c_7565e0cf39c553ea85172f8e95032446 is 50, key is test_row_0/A:col10/1732121693352/Put/seqid=0 2024-11-20T16:54:55,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742411_1587 (size=12154) 2024-11-20T16:54:55,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T16:54:55,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121755529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:55,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121755533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:55,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121755636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:55,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121755646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:55,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:55,773 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120af6be5fbc4c34993b1cbe2e849b7626c_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120af6be5fbc4c34993b1cbe2e849b7626c_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:55,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/75b1a01674eb4a359892a3f01a936b21, store: [table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:54:55,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/75b1a01674eb4a359892a3f01a936b21 is 175, key is test_row_0/A:col10/1732121693352/Put/seqid=0 2024-11-20T16:54:55,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742412_1588 (size=30955) 2024-11-20T16:54:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T16:54:55,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121755845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:55,848 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:55,848 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:55,848 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:55,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121755856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:56,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121756156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:56,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121756166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:56,181 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=102, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/75b1a01674eb4a359892a3f01a936b21 2024-11-20T16:54:56,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/c8ae4612c1dc49d2a531aa63704db985 is 50, key is test_row_0/B:col10/1732121693352/Put/seqid=0 2024-11-20T16:54:56,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742413_1589 (size=12001) 2024-11-20T16:54:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T16:54:56,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33970 deadline: 1732121756416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:56,417 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 7565e0cf39c553ea85172f8e95032446 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:56,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33980 deadline: 1732121756425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:56,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33984 deadline: 1732121756425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:56,429 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 7565e0cf39c553ea85172f8e95032446 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:56,429 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 7565e0cf39c553ea85172f8e95032446 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:54:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:56,603 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/c8ae4612c1dc49d2a531aa63704db985 2024-11-20T16:54:56,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/92b138868ef54d76a2eab3da87464bd9 is 50, key is test_row_0/C:col10/1732121693352/Put/seqid=0 2024-11-20T16:54:56,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742414_1590 (size=12001) 2024-11-20T16:54:56,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121756666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:56,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:56,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:56,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:56,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121756675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:56,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:56,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:56,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:57,015 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/92b138868ef54d76a2eab3da87464bd9 2024-11-20T16:54:57,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/75b1a01674eb4a359892a3f01a936b21 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/75b1a01674eb4a359892a3f01a936b21 2024-11-20T16:54:57,027 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/75b1a01674eb4a359892a3f01a936b21, entries=150, sequenceid=102, filesize=30.2 K 2024-11-20T16:54:57,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/c8ae4612c1dc49d2a531aa63704db985 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/c8ae4612c1dc49d2a531aa63704db985 2024-11-20T16:54:57,031 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/c8ae4612c1dc49d2a531aa63704db985, entries=150, sequenceid=102, filesize=11.7 K 2024-11-20T16:54:57,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/92b138868ef54d76a2eab3da87464bd9 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/92b138868ef54d76a2eab3da87464bd9 2024-11-20T16:54:57,034 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/92b138868ef54d76a2eab3da87464bd9, entries=150, sequenceid=102, filesize=11.7 K 2024-11-20T16:54:57,035 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 7565e0cf39c553ea85172f8e95032446 in 1674ms, sequenceid=102, compaction requested=false 2024-11-20T16:54:57,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:54:57,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:57,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-20T16:54:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-20T16:54:57,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-20T16:54:57,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8270 sec 2024-11-20T16:54:57,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.8310 sec 2024-11-20T16:54:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T16:54:57,357 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:57,357 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:54:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-20T16:54:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T16:54:57,359 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:54:57,359 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:54:57,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:54:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T16:54:57,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-20T16:54:57,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:57,511 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 7565e0cf39c553ea85172f8e95032446 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T16:54:57,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=A 2024-11-20T16:54:57,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:57,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=B 2024-11-20T16:54:57,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:57,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=C 2024-11-20T16:54:57,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:57,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c34b36d1fe4144259ef19be31c8a25c8_7565e0cf39c553ea85172f8e95032446 is 50, key is test_row_0/A:col10/1732121695532/Put/seqid=0 2024-11-20T16:54:57,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742415_1591 (size=12154) 2024-11-20T16:54:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T16:54:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:57,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:57,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121697714, max timestamp=1732121697746Num uniques 859; 2024-11-20T16:54:57,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:57,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121697714, max timestamp=1732121697746Num uniques 859; 2024-11-20T16:54:57,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121697714, max timestamp=1732121697746Num uniques 859; 2024-11-20T16:54:57,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:57,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:54:57,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121757795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:57,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121757800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:57,807 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:57,807 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:57,807 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:57,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:57,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121757906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:57,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:57,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121757916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:57,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:57,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:57,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:57,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:57,923 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c34b36d1fe4144259ef19be31c8a25c8_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c34b36d1fe4144259ef19be31c8a25c8_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:57,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/a42433f197394c97845071bd43463654, store: [table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:54:57,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/a42433f197394c97845071bd43463654 is 175, key is test_row_0/A:col10/1732121695532/Put/seqid=0 2024-11-20T16:54:57,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742416_1592 (size=30955) 2024-11-20T16:54:57,927 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/a42433f197394c97845071bd43463654 2024-11-20T16:54:57,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/69689ebba94740f4821866662188f6b0 is 50, key is test_row_0/B:col10/1732121695532/Put/seqid=0 2024-11-20T16:54:57,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742417_1593 (size=12001) 2024-11-20T16:54:57,935 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/69689ebba94740f4821866662188f6b0 2024-11-20T16:54:57,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/81b8ef84a8ef4250b222eb3c50eb3273 is 50, key is test_row_0/C:col10/1732121695532/Put/seqid=0 2024-11-20T16:54:57,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742418_1594 (size=12001) 2024-11-20T16:54:57,943 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/81b8ef84a8ef4250b222eb3c50eb3273 2024-11-20T16:54:57,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/a42433f197394c97845071bd43463654 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/a42433f197394c97845071bd43463654 2024-11-20T16:54:57,950 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/a42433f197394c97845071bd43463654, entries=150, sequenceid=112, filesize=30.2 K 2024-11-20T16:54:57,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/69689ebba94740f4821866662188f6b0 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/69689ebba94740f4821866662188f6b0 2024-11-20T16:54:57,954 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/69689ebba94740f4821866662188f6b0, entries=150, sequenceid=112, filesize=11.7 K 2024-11-20T16:54:57,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/81b8ef84a8ef4250b222eb3c50eb3273 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/81b8ef84a8ef4250b222eb3c50eb3273 2024-11-20T16:54:57,957 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/81b8ef84a8ef4250b222eb3c50eb3273, entries=150, sequenceid=112, filesize=11.7 K 2024-11-20T16:54:57,958 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7565e0cf39c553ea85172f8e95032446 in 447ms, sequenceid=112, compaction requested=true 2024-11-20T16:54:57,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:54:57,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:57,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-20T16:54:57,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-20T16:54:57,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-20T16:54:57,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 600 msec 2024-11-20T16:54:57,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 605 msec 2024-11-20T16:54:57,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T16:54:57,986 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:54:57,986 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:54:57,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-20T16:54:57,988 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:54:57,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T16:54:57,988 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:54:57,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:54:58,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T16:54:58,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:58,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7565e0cf39c553ea85172f8e95032446 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-20T16:54:58,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=A 2024-11-20T16:54:58,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:58,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=B 2024-11-20T16:54:58,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:58,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=C 2024-11-20T16:54:58,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:54:58,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120635186c4a5d94a7e87be3f0131d0367a_7565e0cf39c553ea85172f8e95032446 is 50, key is test_row_0/A:col10/1732121698118/Put/seqid=0 2024-11-20T16:54:58,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742419_1595 (size=14794) 2024-11-20T16:54:58,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121758135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:58,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:58,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:58,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:58,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:58,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121758136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:58,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T16:54:58,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:58,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:58,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:58,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:58,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:58,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121758246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:58,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121758246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:58,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T16:54:58,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:58,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:58,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:58,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:58,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T16:54:58,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T16:54:58,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:58,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:58,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:58,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:58,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:58,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121758456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:58,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:58,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:58,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121758456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:58,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:58,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:58,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:58,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:58,529 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:54:58,531 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120635186c4a5d94a7e87be3f0131d0367a_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120635186c4a5d94a7e87be3f0131d0367a_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:54:58,532 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/6761962959eb4345ac7c2d4a98ea2e65, store: [table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:54:58,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/6761962959eb4345ac7c2d4a98ea2e65 is 175, key is test_row_0/A:col10/1732121698118/Put/seqid=0 2024-11-20T16:54:58,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742420_1596 (size=39749) 2024-11-20T16:54:58,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T16:54:58,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:58,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:58,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:58,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:58,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T16:54:58,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T16:54:58,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:58,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:58,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:58,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:58,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:58,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:58,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121758766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:58,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121758766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:58,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T16:54:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:58,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:58,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:58,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:58,935 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=149, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/6761962959eb4345ac7c2d4a98ea2e65 2024-11-20T16:54:58,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/8023a131e1b94940aec9d01107298939 is 50, key is test_row_0/B:col10/1732121698118/Put/seqid=0 2024-11-20T16:54:58,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742421_1597 (size=12151) 2024-11-20T16:54:59,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T16:54:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:59,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:59,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T16:54:59,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T16:54:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:59,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:59,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121759276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:59,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:54:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121759276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:54:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:54:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:54:59,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/8023a131e1b94940aec9d01107298939 2024-11-20T16:54:59,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/b22430499496456c96358548badafd51 is 50, key is test_row_0/C:col10/1732121698118/Put/seqid=0 2024-11-20T16:54:59,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742422_1598 (size=12151) 2024-11-20T16:54:59,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T16:54:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:59,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:59,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:59,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T16:54:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:59,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:59,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T16:54:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:54:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:59,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:54:59,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/b22430499496456c96358548badafd51 2024-11-20T16:54:59,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/6761962959eb4345ac7c2d4a98ea2e65 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/6761962959eb4345ac7c2d4a98ea2e65 2024-11-20T16:54:59,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/6761962959eb4345ac7c2d4a98ea2e65, entries=200, sequenceid=149, filesize=38.8 K 2024-11-20T16:54:59,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/8023a131e1b94940aec9d01107298939 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/8023a131e1b94940aec9d01107298939 2024-11-20T16:54:59,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/8023a131e1b94940aec9d01107298939, entries=150, sequenceid=149, filesize=11.9 K 2024-11-20T16:54:59,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/b22430499496456c96358548badafd51 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/b22430499496456c96358548badafd51 2024-11-20T16:54:59,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/b22430499496456c96358548badafd51, entries=150, sequenceid=149, filesize=11.9 K 2024-11-20T16:54:59,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 7565e0cf39c553ea85172f8e95032446 in 1646ms, sequenceid=149, compaction requested=true 2024-11-20T16:54:59,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:54:59,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:54:59,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:59,765 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:54:59,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:54:59,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:54:59,765 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:54:59,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:54:59,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:59,766 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132717 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:54:59,766 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:54:59,766 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/B is initiating minor compaction (all files) 2024-11-20T16:54:59,766 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/A is initiating minor compaction (all files) 2024-11-20T16:54:59,766 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/B in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:59,766 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/A in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:59,766 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/0e15d72d64bb4d8b80bd6ff7c74c766e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/75b1a01674eb4a359892a3f01a936b21, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/a42433f197394c97845071bd43463654, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/6761962959eb4345ac7c2d4a98ea2e65] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=129.6 K 2024-11-20T16:54:59,766 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/072c727a69964d6085cf40ca2233fb16, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/c8ae4612c1dc49d2a531aa63704db985, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/69689ebba94740f4821866662188f6b0, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/8023a131e1b94940aec9d01107298939] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=47.1 K 2024-11-20T16:54:59,766 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:59,766 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/0e15d72d64bb4d8b80bd6ff7c74c766e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/75b1a01674eb4a359892a3f01a936b21, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/a42433f197394c97845071bd43463654, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/6761962959eb4345ac7c2d4a98ea2e65] 2024-11-20T16:54:59,767 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 072c727a69964d6085cf40ca2233fb16, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732121692259 2024-11-20T16:54:59,767 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e15d72d64bb4d8b80bd6ff7c74c766e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732121692259 2024-11-20T16:54:59,767 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75b1a01674eb4a359892a3f01a936b21, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732121693341 2024-11-20T16:54:59,767 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting c8ae4612c1dc49d2a531aa63704db985, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732121693341 2024-11-20T16:54:59,767 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting a42433f197394c97845071bd43463654, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732121695523 2024-11-20T16:54:59,767 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 69689ebba94740f4821866662188f6b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732121695523 2024-11-20T16:54:59,767 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6761962959eb4345ac7c2d4a98ea2e65, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732121697765 2024-11-20T16:54:59,768 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 8023a131e1b94940aec9d01107298939, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732121697765 2024-11-20T16:54:59,779 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:54:59,781 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411209a597008481d450ca0aa21dc2f4b7c6e_7565e0cf39c553ea85172f8e95032446 store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:54:59,781 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#B#compaction#502 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:59,782 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/c9620f0c727e47ebab4b65c8372a3ba1 is 50, key is test_row_0/B:col10/1732121698118/Put/seqid=0 2024-11-20T16:54:59,784 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411209a597008481d450ca0aa21dc2f4b7c6e_7565e0cf39c553ea85172f8e95032446, store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:54:59,784 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209a597008481d450ca0aa21dc2f4b7c6e_7565e0cf39c553ea85172f8e95032446 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:54:59,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742423_1599 (size=12391) 2024-11-20T16:54:59,802 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/c9620f0c727e47ebab4b65c8372a3ba1 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/c9620f0c727e47ebab4b65c8372a3ba1 2024-11-20T16:54:59,806 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/B of 7565e0cf39c553ea85172f8e95032446 into c9620f0c727e47ebab4b65c8372a3ba1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:54:59,806 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:54:59,806 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/B, priority=12, startTime=1732121699765; duration=0sec 2024-11-20T16:54:59,806 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:54:59,806 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:B 2024-11-20T16:54:59,806 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T16:54:59,807 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T16:54:59,807 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/C is initiating minor compaction (all files) 2024-11-20T16:54:59,807 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/C in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:59,807 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/56c666c554a746c19b499e29131ebf34, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/92b138868ef54d76a2eab3da87464bd9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/81b8ef84a8ef4250b222eb3c50eb3273, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/b22430499496456c96358548badafd51] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=47.1 K 2024-11-20T16:54:59,807 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 56c666c554a746c19b499e29131ebf34, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732121692259 2024-11-20T16:54:59,807 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 92b138868ef54d76a2eab3da87464bd9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732121693341 2024-11-20T16:54:59,808 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 81b8ef84a8ef4250b222eb3c50eb3273, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732121695523 2024-11-20T16:54:59,809 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting b22430499496456c96358548badafd51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732121697765 2024-11-20T16:54:59,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742424_1600 (size=4469) 2024-11-20T16:54:59,813 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#A#compaction#503 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:59,813 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/db5791b06bed467dbbf2021707e08014 is 175, key is test_row_0/A:col10/1732121698118/Put/seqid=0 2024-11-20T16:54:59,818 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#C#compaction#504 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:54:59,818 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/c347d37165b34d97a5a5bb8545d91995 is 50, key is test_row_0/C:col10/1732121698118/Put/seqid=0 2024-11-20T16:54:59,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T16:54:59,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:54:59,819 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 7565e0cf39c553ea85172f8e95032446 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-20T16:54:59,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=A 2024-11-20T16:54:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=B 2024-11-20T16:54:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=C 2024-11-20T16:54:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:54:59,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742426_1602 (size=12391) 2024-11-20T16:54:59,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742425_1601 (size=31345) 2024-11-20T16:54:59,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b5b799dce4194e55a55b23e5b31a01a1_7565e0cf39c553ea85172f8e95032446 is 50, key is test_row_0/A:col10/1732121698123/Put/seqid=0 2024-11-20T16:54:59,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742427_1603 (size=12304) 2024-11-20T16:55:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T16:55:00,228 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/db5791b06bed467dbbf2021707e08014 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/db5791b06bed467dbbf2021707e08014 2024-11-20T16:55:00,228 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/c347d37165b34d97a5a5bb8545d91995 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/c347d37165b34d97a5a5bb8545d91995 2024-11-20T16:55:00,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:00,233 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/C of 7565e0cf39c553ea85172f8e95032446 into c347d37165b34d97a5a5bb8545d91995(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:55:00,233 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:00,233 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/C, priority=12, startTime=1732121699765; duration=0sec 2024-11-20T16:55:00,233 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:00,233 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:C 2024-11-20T16:55:00,234 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/A of 7565e0cf39c553ea85172f8e95032446 into db5791b06bed467dbbf2021707e08014(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:55:00,234 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:00,234 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/A, priority=12, startTime=1732121699765; duration=0sec 2024-11-20T16:55:00,235 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:00,235 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:A 2024-11-20T16:55:00,235 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b5b799dce4194e55a55b23e5b31a01a1_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b5b799dce4194e55a55b23e5b31a01a1_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:00,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/0e01444c6fa640ce952fac3da3caef55, store: [table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:00,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/0e01444c6fa640ce952fac3da3caef55 is 175, key is test_row_0/A:col10/1732121698123/Put/seqid=0 2024-11-20T16:55:00,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742428_1604 (size=31105) 2024-11-20T16:55:00,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:00,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:55:00,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121700310, max timestamp=1732121700329Num uniques 859; 2024-11-20T16:55:00,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:55:00,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121700310, max timestamp=1732121700329Num uniques 859; 2024-11-20T16:55:00,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121700310, max timestamp=1732121700329Num uniques 859; 2024-11-20T16:55:00,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:55:00,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:55:00,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121760368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:00,374 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,374 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,374 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:00,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121760372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:00,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:00,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33970 deadline: 1732121760426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:00,427 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 7565e0cf39c553ea85172f8e95032446 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:55:00,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:00,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33980 deadline: 1732121760436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:00,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33984 deadline: 1732121760436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:00,440 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 7565e0cf39c553ea85172f8e95032446 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:55:00,440 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 7565e0cf39c553ea85172f8e95032446 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:55:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:00,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121760476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:00,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:00,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121760486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:00,640 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/0e01444c6fa640ce952fac3da3caef55 2024-11-20T16:55:00,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/f2b7904e49e74172a5ce18e473d1d5e1 is 50, key is test_row_0/B:col10/1732121698123/Put/seqid=0 2024-11-20T16:55:00,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742429_1605 (size=12151) 2024-11-20T16:55:00,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121760686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:00,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:00,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121760696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:00,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:00,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121760996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:00,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:00,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:01,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121761006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:01,049 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/f2b7904e49e74172a5ce18e473d1d5e1 2024-11-20T16:55:01,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/7cbf4a9ef4e3441497466b70d487a04a is 50, key is test_row_0/C:col10/1732121698123/Put/seqid=0 2024-11-20T16:55:01,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742430_1606 (size=12151) 2024-11-20T16:55:01,458 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/7cbf4a9ef4e3441497466b70d487a04a 2024-11-20T16:55:01,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/0e01444c6fa640ce952fac3da3caef55 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/0e01444c6fa640ce952fac3da3caef55 2024-11-20T16:55:01,464 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/0e01444c6fa640ce952fac3da3caef55, entries=150, sequenceid=159, filesize=30.4 K 2024-11-20T16:55:01,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/f2b7904e49e74172a5ce18e473d1d5e1 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/f2b7904e49e74172a5ce18e473d1d5e1 2024-11-20T16:55:01,467 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/f2b7904e49e74172a5ce18e473d1d5e1, entries=150, sequenceid=159, filesize=11.9 K 2024-11-20T16:55:01,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/7cbf4a9ef4e3441497466b70d487a04a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/7cbf4a9ef4e3441497466b70d487a04a 2024-11-20T16:55:01,470 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/7cbf4a9ef4e3441497466b70d487a04a, entries=150, sequenceid=159, filesize=11.9 K 2024-11-20T16:55:01,471 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 7565e0cf39c553ea85172f8e95032446 in 1652ms, sequenceid=159, compaction requested=false 2024-11-20T16:55:01,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:01,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:01,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-20T16:55:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-20T16:55:01,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-20T16:55:01,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4840 sec 2024-11-20T16:55:01,475 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 3.4880 sec 2024-11-20T16:55:01,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:01,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7565e0cf39c553ea85172f8e95032446 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-20T16:55:01,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=A 2024-11-20T16:55:01,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T16:55:01,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=B 2024-11-20T16:55:01,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T16:55:01,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=C 2024-11-20T16:55:01,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T16:55:01,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120be12502514fe46178fb5d3e9e6e8c65a_7565e0cf39c553ea85172f8e95032446 is 50, key is test_row_0/A:col10/1732121701508/Put/seqid=0 2024-11-20T16:55:01,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742431_1607 (size=14794) 2024-11-20T16:55:01,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121761524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:01,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121761525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:01,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121761636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:01,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121761636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:01,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121761846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:01,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:01,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121761846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:01,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:01,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:01,922 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:01,925 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120be12502514fe46178fb5d3e9e6e8c65a_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120be12502514fe46178fb5d3e9e6e8c65a_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:01,926 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/68680c4598ea478e8c8e129c000d6e6b, store: [table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:01,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/68680c4598ea478e8c8e129c000d6e6b is 175, key is test_row_0/A:col10/1732121701508/Put/seqid=0 2024-11-20T16:55:01,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742432_1608 (size=39749) 2024-11-20T16:55:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T16:55:02,146 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:55:02,146 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:55:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-20T16:55:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T16:55:02,148 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:55:02,148 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:55:02,148 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:55:02,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121762156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:02,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121762156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:02,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T16:55:02,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T16:55:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:55:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:02,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:02,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:02,330 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/68680c4598ea478e8c8e129c000d6e6b 2024-11-20T16:55:02,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/184c8debf00545a98398368a4c9b3242 is 50, key is test_row_0/B:col10/1732121701508/Put/seqid=0 2024-11-20T16:55:02,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742433_1609 (size=12151) 2024-11-20T16:55:02,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T16:55:02,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:02,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:55:02,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:02,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:02,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:02,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:02,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T16:55:02,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T16:55:02,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:02,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:55:02,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:02,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:02,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:02,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121762666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:02,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121762666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:02,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/184c8debf00545a98398368a4c9b3242 2024-11-20T16:55:02,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/9a9c79ec42ad472387ecf463fb0d6e1e is 50, key is test_row_0/C:col10/1732121701508/Put/seqid=0 2024-11-20T16:55:02,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742434_1610 (size=12151) 2024-11-20T16:55:02,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T16:55:02,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:02,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:55:02,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:02,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:02,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:02,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T16:55:02,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T16:55:02,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:02,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:55:02,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:02,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:02,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:02,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:03,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T16:55:03,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:03,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:55:03,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:03,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:03,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:03,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/9a9c79ec42ad472387ecf463fb0d6e1e 2024-11-20T16:55:03,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/68680c4598ea478e8c8e129c000d6e6b as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/68680c4598ea478e8c8e129c000d6e6b 2024-11-20T16:55:03,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/68680c4598ea478e8c8e129c000d6e6b, entries=200, sequenceid=199, filesize=38.8 K 2024-11-20T16:55:03,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/184c8debf00545a98398368a4c9b3242 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/184c8debf00545a98398368a4c9b3242 2024-11-20T16:55:03,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/184c8debf00545a98398368a4c9b3242, entries=150, sequenceid=199, filesize=11.9 K 2024-11-20T16:55:03,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/9a9c79ec42ad472387ecf463fb0d6e1e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/9a9c79ec42ad472387ecf463fb0d6e1e 2024-11-20T16:55:03,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/9a9c79ec42ad472387ecf463fb0d6e1e, entries=150, sequenceid=199, filesize=11.9 K 2024-11-20T16:55:03,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 7565e0cf39c553ea85172f8e95032446 in 1656ms, sequenceid=199, compaction requested=true 2024-11-20T16:55:03,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:03,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:55:03,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:03,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:55:03,166 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:55:03,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:03,166 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:55:03,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:55:03,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:55:03,167 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:55:03,167 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/B is initiating minor compaction (all files) 2024-11-20T16:55:03,167 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/B in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:03,167 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/c9620f0c727e47ebab4b65c8372a3ba1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/f2b7904e49e74172a5ce18e473d1d5e1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/184c8debf00545a98398368a4c9b3242] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=35.8 K 2024-11-20T16:55:03,167 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:55:03,168 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/A is initiating minor compaction (all files) 2024-11-20T16:55:03,168 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/A in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:03,168 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/db5791b06bed467dbbf2021707e08014, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/0e01444c6fa640ce952fac3da3caef55, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/68680c4598ea478e8c8e129c000d6e6b] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=99.8 K 2024-11-20T16:55:03,168 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:03,168 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/db5791b06bed467dbbf2021707e08014, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/0e01444c6fa640ce952fac3da3caef55, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/68680c4598ea478e8c8e129c000d6e6b] 2024-11-20T16:55:03,168 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting c9620f0c727e47ebab4b65c8372a3ba1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732121697765 2024-11-20T16:55:03,168 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting db5791b06bed467dbbf2021707e08014, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732121697765 2024-11-20T16:55:03,168 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting f2b7904e49e74172a5ce18e473d1d5e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732121698123 2024-11-20T16:55:03,168 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e01444c6fa640ce952fac3da3caef55, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732121698123 2024-11-20T16:55:03,169 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 184c8debf00545a98398368a4c9b3242, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732121700359 2024-11-20T16:55:03,169 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68680c4598ea478e8c8e129c000d6e6b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732121700359 2024-11-20T16:55:03,173 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:03,175 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#B#compaction#512 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:55:03,175 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/25af895a111b4473942a8b2dfe377e68 is 50, key is test_row_0/B:col10/1732121701508/Put/seqid=0 2024-11-20T16:55:03,175 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120b8908d5b194848f8b8d2e8f142a90654_7565e0cf39c553ea85172f8e95032446 store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:03,177 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120b8908d5b194848f8b8d2e8f142a90654_7565e0cf39c553ea85172f8e95032446, store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:03,177 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b8908d5b194848f8b8d2e8f142a90654_7565e0cf39c553ea85172f8e95032446 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:03,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742435_1611 (size=12493) 2024-11-20T16:55:03,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742436_1612 (size=4469) 2024-11-20T16:55:03,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T16:55:03,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:03,216 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 7565e0cf39c553ea85172f8e95032446 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-20T16:55:03,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=A 2024-11-20T16:55:03,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:55:03,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=B 2024-11-20T16:55:03,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:55:03,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=C 2024-11-20T16:55:03,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:55:03,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112008fd6ac24516416281226cf5983577ce_7565e0cf39c553ea85172f8e95032446 is 50, key is test_row_0/A:col10/1732121701519/Put/seqid=0 2024-11-20T16:55:03,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742437_1613 (size=12304) 2024-11-20T16:55:03,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:03,229 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112008fd6ac24516416281226cf5983577ce_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112008fd6ac24516416281226cf5983577ce_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:03,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/57e1ea4f913c4db1b87fbb164b7f5a14, store: [table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:03,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/57e1ea4f913c4db1b87fbb164b7f5a14 is 175, key is test_row_0/A:col10/1732121701519/Put/seqid=0 2024-11-20T16:55:03,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742438_1614 (size=31105) 2024-11-20T16:55:03,233 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/57e1ea4f913c4db1b87fbb164b7f5a14 2024-11-20T16:55:03,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/4322748d7bff4adc8a9f3ab9a121b9c5 is 50, key is test_row_0/B:col10/1732121701519/Put/seqid=0 2024-11-20T16:55:03,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742439_1615 (size=12151) 2024-11-20T16:55:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T16:55:03,583 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/25af895a111b4473942a8b2dfe377e68 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/25af895a111b4473942a8b2dfe377e68 2024-11-20T16:55:03,587 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/B of 7565e0cf39c553ea85172f8e95032446 into 25af895a111b4473942a8b2dfe377e68(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:55:03,587 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:03,587 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/B, priority=13, startTime=1732121703166; duration=0sec 2024-11-20T16:55:03,587 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:55:03,587 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:B 2024-11-20T16:55:03,587 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:55:03,588 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:55:03,588 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/C is initiating minor compaction (all files) 2024-11-20T16:55:03,588 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/C in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:03,588 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/c347d37165b34d97a5a5bb8545d91995, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/7cbf4a9ef4e3441497466b70d487a04a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/9a9c79ec42ad472387ecf463fb0d6e1e] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=35.8 K 2024-11-20T16:55:03,588 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#A#compaction#511 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:55:03,589 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting c347d37165b34d97a5a5bb8545d91995, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732121697765 2024-11-20T16:55:03,589 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/162741c6de7446c78f04b97a6244ab08 is 175, key is test_row_0/A:col10/1732121701508/Put/seqid=0 2024-11-20T16:55:03,589 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cbf4a9ef4e3441497466b70d487a04a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732121698123 2024-11-20T16:55:03,589 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a9c79ec42ad472387ecf463fb0d6e1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732121700359 2024-11-20T16:55:03,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742440_1616 (size=31447) 2024-11-20T16:55:03,595 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#C#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:55:03,596 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/f3ce8405f1ac4dfab8cba233a375052a is 50, key is test_row_0/C:col10/1732121701508/Put/seqid=0 2024-11-20T16:55:03,597 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/162741c6de7446c78f04b97a6244ab08 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/162741c6de7446c78f04b97a6244ab08 2024-11-20T16:55:03,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742441_1617 (size=12493) 2024-11-20T16:55:03,601 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/A of 7565e0cf39c553ea85172f8e95032446 into 162741c6de7446c78f04b97a6244ab08(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:55:03,601 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:03,601 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/A, priority=13, startTime=1732121703166; duration=0sec 2024-11-20T16:55:03,601 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:03,601 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:A 2024-11-20T16:55:03,641 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/4322748d7bff4adc8a9f3ab9a121b9c5 2024-11-20T16:55:03,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/ba89e77f0677497885471d8a8284fa13 is 50, key is test_row_0/C:col10/1732121701519/Put/seqid=0 2024-11-20T16:55:03,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742442_1618 (size=12151) 2024-11-20T16:55:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:03,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:55:03,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121703723, max timestamp=1732121703733Num uniques 859; 2024-11-20T16:55:03,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121703723, max timestamp=1732121703733Num uniques 859; 2024-11-20T16:55:03,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:55:03,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:55:03,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121703723, max timestamp=1732121703733Num uniques 859; 2024-11-20T16:55:03,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:55:03,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121763778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:03,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121763780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:03,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732121703755, max timestamp=1732121703776Num uniques 1018; 2024-11-20T16:55:03,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T16:55:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:03,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732121703755, max timestamp=1732121703776Num uniques 1018; 2024-11-20T16:55:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:03,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T16:55:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:03,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732121703755, max timestamp=1732121703776Num uniques 1018; 2024-11-20T16:55:03,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T16:55:03,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732121703909, max timestamp=1732121703926Num uniques 1027; 2024-11-20T16:55:03,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-20T16:55:03,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732121703909, max timestamp=1732121703926Num uniques 1027; 2024-11-20T16:55:03,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732121703909, max timestamp=1732121703926Num uniques 1027; 2024-11-20T16:55:03,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-20T16:55:03,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-20T16:55:03,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732121703972, max timestamp=1732121703990Num uniques 1086; 2024-11-20T16:55:03,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732121703972, max timestamp=1732121703990Num uniques 1086; 2024-11-20T16:55:03,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732121703972, max timestamp=1732121703990Num uniques 1086; 2024-11-20T16:55:03,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-20T16:55:03,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-20T16:55:03,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-20T16:55:04,004 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/f3ce8405f1ac4dfab8cba233a375052a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/f3ce8405f1ac4dfab8cba233a375052a 2024-11-20T16:55:04,007 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/C of 7565e0cf39c553ea85172f8e95032446 into f3ce8405f1ac4dfab8cba233a375052a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:55:04,007 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:04,007 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/C, priority=13, startTime=1732121703166; duration=0sec 2024-11-20T16:55:04,007 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:04,008 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:C 2024-11-20T16:55:04,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:04,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121764038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:04,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:04,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:04,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:04,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732121704018, max timestamp=1732121704042Num uniques 1095; 2024-11-20T16:55:04,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-20T16:55:04,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732121704018, max timestamp=1732121704042Num uniques 1095; 2024-11-20T16:55:04,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-20T16:55:04,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732121704018, max timestamp=1732121704042Num uniques 1095; 2024-11-20T16:55:04,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-20T16:55:04,049 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/ba89e77f0677497885471d8a8284fa13 2024-11-20T16:55:04,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/57e1ea4f913c4db1b87fbb164b7f5a14 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/57e1ea4f913c4db1b87fbb164b7f5a14 2024-11-20T16:55:04,056 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/57e1ea4f913c4db1b87fbb164b7f5a14, entries=150, sequenceid=207, filesize=30.4 K 2024-11-20T16:55:04,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/4322748d7bff4adc8a9f3ab9a121b9c5 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/4322748d7bff4adc8a9f3ab9a121b9c5 2024-11-20T16:55:04,061 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/4322748d7bff4adc8a9f3ab9a121b9c5, entries=150, sequenceid=207, filesize=11.9 K 2024-11-20T16:55:04,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/ba89e77f0677497885471d8a8284fa13 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/ba89e77f0677497885471d8a8284fa13 2024-11-20T16:55:04,064 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/ba89e77f0677497885471d8a8284fa13, entries=150, sequenceid=207, filesize=11.9 K 2024-11-20T16:55:04,065 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=46.96 KB/48090 for 7565e0cf39c553ea85172f8e95032446 in 850ms, sequenceid=207, compaction requested=false 2024-11-20T16:55:04,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:04,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:04,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-20T16:55:04,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:04,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7565e0cf39c553ea85172f8e95032446 3/3 column families, dataSize=53.67 KB heapSize=142.55 KB 2024-11-20T16:55:04,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=A 2024-11-20T16:55:04,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-20T16:55:04,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:55:04,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=B 2024-11-20T16:55:04,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:55:04,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=C 2024-11-20T16:55:04,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:55:04,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-20T16:55:04,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9180 sec 2024-11-20T16:55:04,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.9220 sec 2024-11-20T16:55:04,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120068ea9cbe4374d8d8607c2e6ec957864_7565e0cf39c553ea85172f8e95032446 is 50, key is test_row_0/A:col10/1732121704058/Put/seqid=0 2024-11-20T16:55:04,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742443_1619 (size=12454) 2024-11-20T16:55:04,082 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:04,086 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120068ea9cbe4374d8d8607c2e6ec957864_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120068ea9cbe4374d8d8607c2e6ec957864_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:04,087 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/d3035ebb1c204ae0875ea2bcb69313f8, store: [table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:04,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/d3035ebb1c204ae0875ea2bcb69313f8 is 175, key is test_row_0/A:col10/1732121704058/Put/seqid=0 2024-11-20T16:55:04,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742444_1620 (size=31255) 2024-11-20T16:55:04,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121704149, max timestamp=1732121704161Num uniques 859; 2024-11-20T16:55:04,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:55:04,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121704149, max timestamp=1732121704161Num uniques 859; 2024-11-20T16:55:04,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732121704149, max timestamp=1732121704161Num uniques 859; 2024-11-20T16:55:04,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:55:04,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T16:55:04,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121764202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:04,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121764207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T16:55:04,296 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:55:04,296 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:55:04,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-20T16:55:04,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T16:55:04,298 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:55:04,298 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:55:04,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:55:04,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:04,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121764316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:04,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:04,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:04,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121764316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:04,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:04,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:04,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T16:55:04,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T16:55:04,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:04,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:55:04,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:04,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:04,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:04,491 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/d3035ebb1c204ae0875ea2bcb69313f8 2024-11-20T16:55:04,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/4c4016c0625c486897a28297948a9c05 is 50, key is test_row_0/B:col10/1732121704058/Put/seqid=0 2024-11-20T16:55:04,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742445_1621 (size=12301) 2024-11-20T16:55:04,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/4c4016c0625c486897a28297948a9c05 2024-11-20T16:55:04,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/29b5443c9a6244be84cffc62e28cea15 is 50, key is test_row_0/C:col10/1732121704058/Put/seqid=0 2024-11-20T16:55:04,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742446_1622 (size=12301) 2024-11-20T16:55:04,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121764525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:04,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:04,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121764526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:04,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:04,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:04,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:04,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:04,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:04,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T16:55:04,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:04,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:55:04,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:04,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:04,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T16:55:04,674 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T16:55:04,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T16:55:04,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:04,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:55:04,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:04,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:04,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:04,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:04,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121764836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:04,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:04,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121764836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:04,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T16:55:04,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:04,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:55:04,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:04,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:04,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:04,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:04,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/29b5443c9a6244be84cffc62e28cea15 2024-11-20T16:55:04,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/d3035ebb1c204ae0875ea2bcb69313f8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/d3035ebb1c204ae0875ea2bcb69313f8 2024-11-20T16:55:04,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/d3035ebb1c204ae0875ea2bcb69313f8, entries=150, sequenceid=303, filesize=30.5 K 2024-11-20T16:55:04,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/4c4016c0625c486897a28297948a9c05 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/4c4016c0625c486897a28297948a9c05 2024-11-20T16:55:04,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/4c4016c0625c486897a28297948a9c05, entries=150, sequenceid=303, filesize=12.0 K 2024-11-20T16:55:04,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/29b5443c9a6244be84cffc62e28cea15 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/29b5443c9a6244be84cffc62e28cea15 2024-11-20T16:55:04,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/29b5443c9a6244be84cffc62e28cea15, entries=150, sequenceid=303, filesize=12.0 K 2024-11-20T16:55:04,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~142.50 KB/145920, currentSize=134.18 KB/137400 for 7565e0cf39c553ea85172f8e95032446 in 858ms, sequenceid=303, compaction requested=true 2024-11-20T16:55:04,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:04,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:55:04,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:04,923 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:55:04,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:55:04,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:04,923 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:55:04,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:55:04,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:55:04,924 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93807 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:55:04,924 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/A is initiating minor compaction (all files) 2024-11-20T16:55:04,924 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:55:04,924 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/A in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:04,924 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/B is initiating minor compaction (all files) 2024-11-20T16:55:04,924 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/B in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:04,924 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/162741c6de7446c78f04b97a6244ab08, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/57e1ea4f913c4db1b87fbb164b7f5a14, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/d3035ebb1c204ae0875ea2bcb69313f8] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=91.6 K 2024-11-20T16:55:04,924 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:04,924 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/25af895a111b4473942a8b2dfe377e68, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/4322748d7bff4adc8a9f3ab9a121b9c5, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/4c4016c0625c486897a28297948a9c05] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=36.1 K 2024-11-20T16:55:04,924 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/162741c6de7446c78f04b97a6244ab08, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/57e1ea4f913c4db1b87fbb164b7f5a14, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/d3035ebb1c204ae0875ea2bcb69313f8] 2024-11-20T16:55:04,924 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 25af895a111b4473942a8b2dfe377e68, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732121700359 2024-11-20T16:55:04,924 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 162741c6de7446c78f04b97a6244ab08, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732121700359 2024-11-20T16:55:04,925 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 4322748d7bff4adc8a9f3ab9a121b9c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732121701519 2024-11-20T16:55:04,925 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57e1ea4f913c4db1b87fbb164b7f5a14, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732121701519 2024-11-20T16:55:04,925 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c4016c0625c486897a28297948a9c05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732121704051 2024-11-20T16:55:04,925 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3035ebb1c204ae0875ea2bcb69313f8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732121704051 2024-11-20T16:55:04,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T16:55:04,929 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:04,930 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#B#compaction#520 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:55:04,930 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120f2e0892d9da34ac2a759ab9273e956ff_7565e0cf39c553ea85172f8e95032446 store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:04,931 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/22610380ced14960b76beaae7e3ba39d is 50, key is test_row_0/B:col10/1732121704058/Put/seqid=0 2024-11-20T16:55:04,933 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120f2e0892d9da34ac2a759ab9273e956ff_7565e0cf39c553ea85172f8e95032446, store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:04,933 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f2e0892d9da34ac2a759ab9273e956ff_7565e0cf39c553ea85172f8e95032446 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:04,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742448_1624 (size=4469) 2024-11-20T16:55:04,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742447_1623 (size=12745) 2024-11-20T16:55:05,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T16:55:05,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:05,060 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 7565e0cf39c553ea85172f8e95032446 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-20T16:55:05,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=A 2024-11-20T16:55:05,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:55:05,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=B 2024-11-20T16:55:05,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:55:05,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=C 2024-11-20T16:55:05,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:55:05,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112057ee5b59d6c044c093db0fc547b58b4b_7565e0cf39c553ea85172f8e95032446 is 50, key is test_row_0/A:col10/1732121704195/Put/seqid=0 2024-11-20T16:55:05,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742449_1625 (size=12454) 2024-11-20T16:55:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:05,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:55:05,361 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#A#compaction#521 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:55:05,361 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/b7809fabdd534e55ba678476a751c625 is 175, key is test_row_0/A:col10/1732121704058/Put/seqid=0 2024-11-20T16:55:05,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121765360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:05,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:05,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:05,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:05,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121765365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:05,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:05,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:05,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:05,368 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/22610380ced14960b76beaae7e3ba39d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/22610380ced14960b76beaae7e3ba39d 2024-11-20T16:55:05,371 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/B of 7565e0cf39c553ea85172f8e95032446 into 22610380ced14960b76beaae7e3ba39d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:55:05,371 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:05,372 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/B, priority=13, startTime=1732121704923; duration=0sec 2024-11-20T16:55:05,372 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:55:05,372 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:B 2024-11-20T16:55:05,372 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:55:05,372 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:55:05,372 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/C is initiating minor compaction (all files) 2024-11-20T16:55:05,373 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/C in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:05,373 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/f3ce8405f1ac4dfab8cba233a375052a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/ba89e77f0677497885471d8a8284fa13, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/29b5443c9a6244be84cffc62e28cea15] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=36.1 K 2024-11-20T16:55:05,373 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting f3ce8405f1ac4dfab8cba233a375052a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732121700359 2024-11-20T16:55:05,373 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting ba89e77f0677497885471d8a8284fa13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732121701519 2024-11-20T16:55:05,373 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 29b5443c9a6244be84cffc62e28cea15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732121704051 2024-11-20T16:55:05,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742450_1626 (size=31699) 2024-11-20T16:55:05,379 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#C#compaction#523 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:55:05,380 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/81bf03b6135849cdb1e9b8648f075607 is 50, key is test_row_0/C:col10/1732121704058/Put/seqid=0 2024-11-20T16:55:05,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742451_1627 (size=12745) 2024-11-20T16:55:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T16:55:05,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:05,478 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112057ee5b59d6c044c093db0fc547b58b4b_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112057ee5b59d6c044c093db0fc547b58b4b_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:05,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/9ecf5d1ff1f9415595541b52142e523f, store: [table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:05,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/9ecf5d1ff1f9415595541b52142e523f is 175, key is test_row_0/A:col10/1732121704195/Put/seqid=0 2024-11-20T16:55:05,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121765476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:05,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121765476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:05,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:05,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:05,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:05,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742452_1628 (size=31255) 2024-11-20T16:55:05,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121765686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:05,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:05,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121765686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:05,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:05,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:05,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:05,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:05,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:05,777 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/b7809fabdd534e55ba678476a751c625 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/b7809fabdd534e55ba678476a751c625 2024-11-20T16:55:05,780 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/A of 7565e0cf39c553ea85172f8e95032446 into b7809fabdd534e55ba678476a751c625(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:55:05,780 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:05,780 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/A, priority=13, startTime=1732121704923; duration=0sec 2024-11-20T16:55:05,780 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:05,780 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:A 2024-11-20T16:55:05,788 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/81bf03b6135849cdb1e9b8648f075607 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/81bf03b6135849cdb1e9b8648f075607 2024-11-20T16:55:05,791 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/C of 7565e0cf39c553ea85172f8e95032446 into 81bf03b6135849cdb1e9b8648f075607(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:55:05,791 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:05,791 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/C, priority=13, startTime=1732121704923; duration=0sec 2024-11-20T16:55:05,791 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:05,791 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:C 2024-11-20T16:55:05,882 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/9ecf5d1ff1f9415595541b52142e523f 2024-11-20T16:55:05,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/8dc8287bcdcf4dd6baafec240b355859 is 50, key is test_row_0/B:col10/1732121704195/Put/seqid=0 2024-11-20T16:55:05,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742453_1629 (size=12301) 2024-11-20T16:55:05,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121765996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:05,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121765996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:06,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:06,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:06,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:06,294 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/8dc8287bcdcf4dd6baafec240b355859 2024-11-20T16:55:06,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/0a90726202ad4f90a604e46e26b3d302 is 50, key is test_row_0/C:col10/1732121704195/Put/seqid=0 2024-11-20T16:55:06,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742454_1630 (size=12301) 2024-11-20T16:55:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T16:55:06,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121766506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:06,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121766506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:06,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:06,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:06,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:06,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:06,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:06,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:06,704 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/0a90726202ad4f90a604e46e26b3d302 2024-11-20T16:55:06,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/9ecf5d1ff1f9415595541b52142e523f as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/9ecf5d1ff1f9415595541b52142e523f 2024-11-20T16:55:06,710 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/9ecf5d1ff1f9415595541b52142e523f, entries=150, sequenceid=339, filesize=30.5 K 2024-11-20T16:55:06,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/8dc8287bcdcf4dd6baafec240b355859 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/8dc8287bcdcf4dd6baafec240b355859 2024-11-20T16:55:06,713 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/8dc8287bcdcf4dd6baafec240b355859, entries=150, sequenceid=339, filesize=12.0 K 2024-11-20T16:55:06,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/0a90726202ad4f90a604e46e26b3d302 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/0a90726202ad4f90a604e46e26b3d302 2024-11-20T16:55:06,718 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/0a90726202ad4f90a604e46e26b3d302, entries=150, sequenceid=339, filesize=12.0 K 2024-11-20T16:55:06,718 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 7565e0cf39c553ea85172f8e95032446 in 1658ms, sequenceid=339, compaction requested=false 2024-11-20T16:55:06,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:06,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:06,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-20T16:55:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-20T16:55:06,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-20T16:55:06,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4220 sec 2024-11-20T16:55:06,723 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 2.4260 sec 2024-11-20T16:55:07,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:07,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7565e0cf39c553ea85172f8e95032446 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T16:55:07,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=A 2024-11-20T16:55:07,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:55:07,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=B 2024-11-20T16:55:07,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:55:07,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=C 2024-11-20T16:55:07,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:55:07,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a1f9e23588274520bb805b41e07b4ba3_7565e0cf39c553ea85172f8e95032446 is 50, key is test_row_0/A:col10/1732121707527/Put/seqid=0 2024-11-20T16:55:07,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742455_1631 (size=14994) 2024-11-20T16:55:07,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121767561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:07,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121767562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:07,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:07,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:07,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:07,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:07,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:07,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:07,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121767676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:07,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121767676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:07,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:07,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121767886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:07,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:07,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121767886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:07,940 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:07,942 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a1f9e23588274520bb805b41e07b4ba3_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a1f9e23588274520bb805b41e07b4ba3_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:07,943 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/62d5faf4dda549e0b291b74c5ab1e1b4, store: [table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:07,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/62d5faf4dda549e0b291b74c5ab1e1b4 is 175, key is test_row_0/A:col10/1732121707527/Put/seqid=0 2024-11-20T16:55:07,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742456_1632 (size=39949) 2024-11-20T16:55:08,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:08,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121768196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:08,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:08,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121768196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:08,346 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/62d5faf4dda549e0b291b74c5ab1e1b4 2024-11-20T16:55:08,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/e8f977d25a664c9fb0c9ff26b5d39524 is 50, key is test_row_0/B:col10/1732121707527/Put/seqid=0 2024-11-20T16:55:08,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742457_1633 (size=12301) 2024-11-20T16:55:08,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T16:55:08,466 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:55:08,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:55:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-20T16:55:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T16:55:08,468 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:55:08,469 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:55:08,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:55:08,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T16:55:08,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-20T16:55:08,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:08,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:55:08,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:08,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:08,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:08,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121768706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:08,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:08,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121768706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:08,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/e8f977d25a664c9fb0c9ff26b5d39524 2024-11-20T16:55:08,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/f2fc0fb6ccdf496195ac44ea8dcda5dd is 50, key is test_row_0/C:col10/1732121707527/Put/seqid=0 2024-11-20T16:55:08,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742458_1634 (size=12301) 2024-11-20T16:55:08,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-20T16:55:08,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:08,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:55:08,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:08,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:08,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:08,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:08,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T16:55:08,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-20T16:55:08,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:08,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:55:08,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:08,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:08,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:09,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-20T16:55:09,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:09,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:55:09,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:09,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:09,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T16:55:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T16:55:09,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/f2fc0fb6ccdf496195ac44ea8dcda5dd 2024-11-20T16:55:09,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/62d5faf4dda549e0b291b74c5ab1e1b4 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/62d5faf4dda549e0b291b74c5ab1e1b4 2024-11-20T16:55:09,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/62d5faf4dda549e0b291b74c5ab1e1b4, entries=200, sequenceid=354, filesize=39.0 K 2024-11-20T16:55:09,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/e8f977d25a664c9fb0c9ff26b5d39524 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/e8f977d25a664c9fb0c9ff26b5d39524 2024-11-20T16:55:09,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/e8f977d25a664c9fb0c9ff26b5d39524, entries=150, sequenceid=354, filesize=12.0 K 2024-11-20T16:55:09,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/f2fc0fb6ccdf496195ac44ea8dcda5dd as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/f2fc0fb6ccdf496195ac44ea8dcda5dd 2024-11-20T16:55:09,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/f2fc0fb6ccdf496195ac44ea8dcda5dd, entries=150, sequenceid=354, filesize=12.0 K 2024-11-20T16:55:09,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7565e0cf39c553ea85172f8e95032446 in 1650ms, sequenceid=354, compaction requested=true 2024-11-20T16:55:09,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:09,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:55:09,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:09,178 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:55:09,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:55:09,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:09,178 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:55:09,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:55:09,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:55:09,179 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:55:09,179 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:55:09,179 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/A is initiating minor compaction (all files) 2024-11-20T16:55:09,179 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/B is initiating minor compaction (all files) 2024-11-20T16:55:09,179 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/A in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:09,179 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/B in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:09,179 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/b7809fabdd534e55ba678476a751c625, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/9ecf5d1ff1f9415595541b52142e523f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/62d5faf4dda549e0b291b74c5ab1e1b4] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=100.5 K 2024-11-20T16:55:09,179 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/22610380ced14960b76beaae7e3ba39d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/8dc8287bcdcf4dd6baafec240b355859, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/e8f977d25a664c9fb0c9ff26b5d39524] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=36.5 K 2024-11-20T16:55:09,179 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:09,179 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/b7809fabdd534e55ba678476a751c625, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/9ecf5d1ff1f9415595541b52142e523f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/62d5faf4dda549e0b291b74c5ab1e1b4] 2024-11-20T16:55:09,179 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 22610380ced14960b76beaae7e3ba39d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732121704051 2024-11-20T16:55:09,180 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7809fabdd534e55ba678476a751c625, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732121704051 2024-11-20T16:55:09,180 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dc8287bcdcf4dd6baafec240b355859, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732121704195 2024-11-20T16:55:09,180 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting e8f977d25a664c9fb0c9ff26b5d39524, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732121705363 2024-11-20T16:55:09,180 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ecf5d1ff1f9415595541b52142e523f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732121704195 2024-11-20T16:55:09,181 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62d5faf4dda549e0b291b74c5ab1e1b4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732121705359 2024-11-20T16:55:09,185 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#B#compaction#529 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:55:09,186 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/6d9f6d44134b4718b08b899c577db7b7 is 50, key is test_row_0/B:col10/1732121707527/Put/seqid=0 2024-11-20T16:55:09,187 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:09,196 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120425529247e9347f2bb0b642ce01494eb_7565e0cf39c553ea85172f8e95032446 store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:09,198 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120425529247e9347f2bb0b642ce01494eb_7565e0cf39c553ea85172f8e95032446, store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:09,198 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120425529247e9347f2bb0b642ce01494eb_7565e0cf39c553ea85172f8e95032446 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:09,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742459_1635 (size=12847) 2024-11-20T16:55:09,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742460_1636 (size=4469) 2024-11-20T16:55:09,205 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#A#compaction#530 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:55:09,205 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/058873af73214715a57b789f963e94ca is 175, key is test_row_0/A:col10/1732121707527/Put/seqid=0 2024-11-20T16:55:09,206 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/6d9f6d44134b4718b08b899c577db7b7 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/6d9f6d44134b4718b08b899c577db7b7 2024-11-20T16:55:09,210 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/B of 7565e0cf39c553ea85172f8e95032446 into 6d9f6d44134b4718b08b899c577db7b7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:55:09,210 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:09,210 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/B, priority=13, startTime=1732121709178; duration=0sec 2024-11-20T16:55:09,210 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:55:09,210 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:B 2024-11-20T16:55:09,210 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:55:09,212 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:55:09,212 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/C is initiating minor compaction (all files) 2024-11-20T16:55:09,212 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/C in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:09,212 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/81bf03b6135849cdb1e9b8648f075607, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/0a90726202ad4f90a604e46e26b3d302, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/f2fc0fb6ccdf496195ac44ea8dcda5dd] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=36.5 K 2024-11-20T16:55:09,213 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 81bf03b6135849cdb1e9b8648f075607, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732121704051 2024-11-20T16:55:09,213 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a90726202ad4f90a604e46e26b3d302, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732121704195 2024-11-20T16:55:09,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742461_1637 (size=31801) 2024-11-20T16:55:09,214 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting f2fc0fb6ccdf496195ac44ea8dcda5dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732121705363 2024-11-20T16:55:09,220 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/058873af73214715a57b789f963e94ca as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/058873af73214715a57b789f963e94ca 2024-11-20T16:55:09,228 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/A of 7565e0cf39c553ea85172f8e95032446 into 058873af73214715a57b789f963e94ca(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:55:09,228 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:09,228 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/A, priority=13, startTime=1732121709178; duration=0sec 2024-11-20T16:55:09,228 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:09,228 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:A 2024-11-20T16:55:09,228 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#C#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:55:09,229 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/f1ab919755ec4a019eac8fc10deeeea8 is 50, key is test_row_0/C:col10/1732121707527/Put/seqid=0 2024-11-20T16:55:09,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-20T16:55:09,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:09,231 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing 7565e0cf39c553ea85172f8e95032446 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T16:55:09,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=A 2024-11-20T16:55:09,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:55:09,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=B 2024-11-20T16:55:09,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:55:09,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=C 2024-11-20T16:55:09,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:55:09,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742462_1638 (size=12847) 2024-11-20T16:55:09,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205d3ad214f0f84593a66efa25543bfb7e_7565e0cf39c553ea85172f8e95032446 is 50, key is test_row_0/A:col10/1732121707561/Put/seqid=0 2024-11-20T16:55:09,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742463_1639 (size=12454) 2024-11-20T16:55:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T16:55:09,638 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/f1ab919755ec4a019eac8fc10deeeea8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/f1ab919755ec4a019eac8fc10deeeea8 2024-11-20T16:55:09,641 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/C of 7565e0cf39c553ea85172f8e95032446 into f1ab919755ec4a019eac8fc10deeeea8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:55:09,641 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:09,641 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/C, priority=13, startTime=1732121709178; duration=0sec 2024-11-20T16:55:09,641 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:09,641 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:C 2024-11-20T16:55:09,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,647 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205d3ad214f0f84593a66efa25543bfb7e_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205d3ad214f0f84593a66efa25543bfb7e_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:09,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/eeeaf6b2c28c4642ba58a483875b0d1a, store: [table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:09,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/eeeaf6b2c28c4642ba58a483875b0d1a is 175, key is test_row_0/A:col10/1732121707561/Put/seqid=0 2024-11-20T16:55:09,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742464_1640 (size=31255) 2024-11-20T16:55:09,656 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=376, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/eeeaf6b2c28c4642ba58a483875b0d1a 2024-11-20T16:55:09,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/b5bc1429ba28436ca991e333a4e3804b is 50, key is test_row_0/B:col10/1732121707561/Put/seqid=0 2024-11-20T16:55:09,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742465_1641 (size=12301) 2024-11-20T16:55:09,670 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/b5bc1429ba28436ca991e333a4e3804b 2024-11-20T16:55:09,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/6716f130c6ac479da13ab5993c0da67e is 50, key is test_row_0/C:col10/1732121707561/Put/seqid=0 2024-11-20T16:55:09,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742466_1642 (size=12301) 2024-11-20T16:55:09,680 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/6716f130c6ac479da13ab5993c0da67e 2024-11-20T16:55:09,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/eeeaf6b2c28c4642ba58a483875b0d1a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/eeeaf6b2c28c4642ba58a483875b0d1a 2024-11-20T16:55:09,686 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/eeeaf6b2c28c4642ba58a483875b0d1a, entries=150, sequenceid=376, filesize=30.5 K 2024-11-20T16:55:09,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/b5bc1429ba28436ca991e333a4e3804b as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/b5bc1429ba28436ca991e333a4e3804b 2024-11-20T16:55:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,690 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/b5bc1429ba28436ca991e333a4e3804b, entries=150, sequenceid=376, filesize=12.0 K 2024-11-20T16:55:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/6716f130c6ac479da13ab5993c0da67e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/6716f130c6ac479da13ab5993c0da67e 2024-11-20T16:55:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,693 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/6716f130c6ac479da13ab5993c0da67e, entries=150, sequenceid=376, filesize=12.0 K 2024-11-20T16:55:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,694 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 7565e0cf39c553ea85172f8e95032446 in 463ms, sequenceid=376, compaction requested=false 2024-11-20T16:55:09,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:09,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:09,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-20T16:55:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-20T16:55:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-20T16:55:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2260 sec 2024-11-20T16:55:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 1.2300 sec 2024-11-20T16:55:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:09,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7565e0cf39c553ea85172f8e95032446 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T16:55:09,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=A 2024-11-20T16:55:09,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:55:09,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=B 2024-11-20T16:55:09,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:55:09,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=C 2024-11-20T16:55:09,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:55:09,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ec8b70c39a574ad8b9bf8a144f79e526_7565e0cf39c553ea85172f8e95032446 is 50, key is test_row_0/A:col10/1732121709756/Put/seqid=0 2024-11-20T16:55:09,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742467_1643 (size=12454) 2024-11-20T16:55:09,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121709793, max timestamp=1732121709805Num uniques 859; 2024-11-20T16:55:09,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121709793, max timestamp=1732121709805Num uniques 859; 2024-11-20T16:55:09,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732121709793, max timestamp=1732121709805Num uniques 859; 2024-11-20T16:55:09,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:55:09,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:55:09,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b60eac9bda43:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T16:55:09,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121769868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:09,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121769869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:09,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:09,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121769976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:09,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:09,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121769986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:10,167 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:10,170 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ec8b70c39a574ad8b9bf8a144f79e526_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ec8b70c39a574ad8b9bf8a144f79e526_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:10,171 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/4ff11cb613da4209a0f6bed099df3c49, store: [table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:10,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/4ff11cb613da4209a0f6bed099df3c49 is 175, key is test_row_0/A:col10/1732121709756/Put/seqid=0 2024-11-20T16:55:10,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742468_1644 (size=31255) 2024-11-20T16:55:10,174 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=387, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/4ff11cb613da4209a0f6bed099df3c49 2024-11-20T16:55:10,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/fddee825f45c4ea3b3937ef65dacf11e is 50, key is test_row_0/B:col10/1732121709756/Put/seqid=0 2024-11-20T16:55:10,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742469_1645 (size=12301) 2024-11-20T16:55:10,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:10,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121770185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:10,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:10,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:10,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:10,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121770196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:10,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33984 deadline: 1732121770476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:10,478 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 7565e0cf39c553ea85172f8e95032446 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:55:10,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:10,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:10,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:10,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121770496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:10,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121770506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:10,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:10,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:10,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:10,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33970 deadline: 1732121770515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:10,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33980 deadline: 1732121770516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:10,519 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_2' in 7565e0cf39c553ea85172f8e95032446 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:55:10,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:10,520 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b60eac9bda43,33929,1732121526384 for 'test_row_1' in 7565e0cf39c553ea85172f8e95032446 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T16:55:10,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:10,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:10,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:10,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:10,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:10,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/fddee825f45c4ea3b3937ef65dacf11e 2024-11-20T16:55:10,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/07960884c4e94e269e7d8e914464bb74 is 50, key is test_row_0/C:col10/1732121709756/Put/seqid=0 2024-11-20T16:55:10,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742470_1646 (size=12301) 2024-11-20T16:55:10,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/07960884c4e94e269e7d8e914464bb74 2024-11-20T16:55:10,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/4ff11cb613da4209a0f6bed099df3c49 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/4ff11cb613da4209a0f6bed099df3c49 2024-11-20T16:55:10,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/4ff11cb613da4209a0f6bed099df3c49, entries=150, sequenceid=387, filesize=30.5 K 2024-11-20T16:55:10,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/fddee825f45c4ea3b3937ef65dacf11e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/fddee825f45c4ea3b3937ef65dacf11e 2024-11-20T16:55:10,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/fddee825f45c4ea3b3937ef65dacf11e, entries=150, sequenceid=387, filesize=12.0 K 2024-11-20T16:55:10,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/07960884c4e94e269e7d8e914464bb74 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/07960884c4e94e269e7d8e914464bb74 2024-11-20T16:55:10,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/07960884c4e94e269e7d8e914464bb74, entries=150, sequenceid=387, filesize=12.0 K 2024-11-20T16:55:10,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7565e0cf39c553ea85172f8e95032446 in 846ms, sequenceid=387, compaction requested=true 2024-11-20T16:55:10,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:10,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:55:10,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:10,603 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:55:10,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:55:10,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:10,603 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:55:10,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:55:10,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:55:10,604 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:55:10,604 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:55:10,604 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/B is initiating minor compaction (all files) 2024-11-20T16:55:10,604 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/A is initiating minor compaction (all files) 2024-11-20T16:55:10,604 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/B in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:10,604 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/A in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:10,604 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/6d9f6d44134b4718b08b899c577db7b7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/b5bc1429ba28436ca991e333a4e3804b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/fddee825f45c4ea3b3937ef65dacf11e] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=36.6 K 2024-11-20T16:55:10,604 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/058873af73214715a57b789f963e94ca, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/eeeaf6b2c28c4642ba58a483875b0d1a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/4ff11cb613da4209a0f6bed099df3c49] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=92.1 K 2024-11-20T16:55:10,604 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:10,604 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/058873af73214715a57b789f963e94ca, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/eeeaf6b2c28c4642ba58a483875b0d1a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/4ff11cb613da4209a0f6bed099df3c49] 2024-11-20T16:55:10,604 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d9f6d44134b4718b08b899c577db7b7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732121705363 2024-11-20T16:55:10,604 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 058873af73214715a57b789f963e94ca, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732121705363 2024-11-20T16:55:10,604 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting b5bc1429ba28436ca991e333a4e3804b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732121707555 2024-11-20T16:55:10,604 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeeaf6b2c28c4642ba58a483875b0d1a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732121707555 2024-11-20T16:55:10,605 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting fddee825f45c4ea3b3937ef65dacf11e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732121709749 2024-11-20T16:55:10,605 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ff11cb613da4209a0f6bed099df3c49, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732121709749 2024-11-20T16:55:10,611 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#B#compaction#538 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:55:10,611 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/0280f43e249e429b8d5634cbea52b7d8 is 50, key is test_row_0/B:col10/1732121709756/Put/seqid=0 2024-11-20T16:55:10,612 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:10,613 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411208f61387471f2411889b1cd5dc76b29f8_7565e0cf39c553ea85172f8e95032446 store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:10,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742471_1647 (size=12949) 2024-11-20T16:55:10,615 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411208f61387471f2411889b1cd5dc76b29f8_7565e0cf39c553ea85172f8e95032446, store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T16:55:10,616 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208f61387471f2411889b1cd5dc76b29f8_7565e0cf39c553ea85172f8e95032446 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:10,616 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:55:10,616 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T16:55:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-11-20T16:55:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-20T16:55:10,619 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T16:55:10,620 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T16:55:10,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T16:55:10,623 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/0280f43e249e429b8d5634cbea52b7d8 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/0280f43e249e429b8d5634cbea52b7d8 2024-11-20T16:55:10,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742472_1648 (size=4469) 2024-11-20T16:55:10,624 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#A#compaction#539 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:55:10,625 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/12913cd139fb45bb8fe0e776322848bf is 175, key is test_row_0/A:col10/1732121709756/Put/seqid=0 2024-11-20T16:55:10,628 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/B of 7565e0cf39c553ea85172f8e95032446 into 0280f43e249e429b8d5634cbea52b7d8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:55:10,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742473_1649 (size=31903) 2024-11-20T16:55:10,628 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:10,628 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/B, priority=13, startTime=1732121710603; duration=0sec 2024-11-20T16:55:10,628 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:55:10,628 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:B 2024-11-20T16:55:10,628 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:55:10,629 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:55:10,629 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/C is initiating minor compaction (all files) 2024-11-20T16:55:10,629 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/C in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:10,629 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/f1ab919755ec4a019eac8fc10deeeea8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/6716f130c6ac479da13ab5993c0da67e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/07960884c4e94e269e7d8e914464bb74] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=36.6 K 2024-11-20T16:55:10,629 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting f1ab919755ec4a019eac8fc10deeeea8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732121705363 2024-11-20T16:55:10,629 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 6716f130c6ac479da13ab5993c0da67e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732121707555 2024-11-20T16:55:10,629 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 07960884c4e94e269e7d8e914464bb74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732121709749 2024-11-20T16:55:10,633 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#C#compaction#540 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:55:10,634 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/b1ad589f6b3946058f2b568c1afbc2ab is 50, key is test_row_0/C:col10/1732121709756/Put/seqid=0 2024-11-20T16:55:10,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742474_1650 (size=12949) 2024-11-20T16:55:10,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-20T16:55:10,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33929 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-20T16:55:10,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:10,772 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing 7565e0cf39c553ea85172f8e95032446 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-20T16:55:10,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=A 2024-11-20T16:55:10,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:55:10,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=B 2024-11-20T16:55:10,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:55:10,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=C 2024-11-20T16:55:10,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T16:55:10,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f14228cc7e644c5390b51e0e6c15fa1b_7565e0cf39c553ea85172f8e95032446 is 50, key is test_row_0/A:col10/1732121709867/Put/seqid=0 2024-11-20T16:55:10,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742475_1651 (size=12454) 2024-11-20T16:55:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-20T16:55:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:11,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. as already flushing 2024-11-20T16:55:11,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121771024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:11,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:11,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:11,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:11,031 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/12913cd139fb45bb8fe0e776322848bf as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/12913cd139fb45bb8fe0e776322848bf 2024-11-20T16:55:11,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121771026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:11,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:11,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:11,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:11,035 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/A of 7565e0cf39c553ea85172f8e95032446 into 12913cd139fb45bb8fe0e776322848bf(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:55:11,035 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:11,035 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/A, priority=13, startTime=1732121710603; duration=0sec 2024-11-20T16:55:11,035 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:11,035 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:A 2024-11-20T16:55:11,040 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/b1ad589f6b3946058f2b568c1afbc2ab as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/b1ad589f6b3946058f2b568c1afbc2ab 2024-11-20T16:55:11,044 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/C of 7565e0cf39c553ea85172f8e95032446 into b1ad589f6b3946058f2b568c1afbc2ab(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:55:11,044 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:11,044 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/C, priority=13, startTime=1732121710603; duration=0sec 2024-11-20T16:55:11,044 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:11,044 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:C 2024-11-20T16:55:11,059 INFO [Thread-2547 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2547. 2024-11-20T16:55:11,060 DEBUG [Thread-2547 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:55:11,060 DEBUG [Thread-2547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:11,060 DEBUG [Thread-2547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:11,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:55:11,060 INFO [Thread-2543 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2543. 2024-11-20T16:55:11,060 DEBUG [Thread-2543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:55:11,060 DEBUG [Thread-2543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:11,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:55:11,060 DEBUG [Thread-2543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:11,061 INFO [Thread-2539 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2539. 2024-11-20T16:55:11,061 DEBUG [Thread-2539 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:55:11,061 DEBUG [Thread-2539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:11,061 DEBUG [Thread-2539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:11,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:55:11,063 INFO [Thread-2541 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2541. 2024-11-20T16:55:11,063 DEBUG [Thread-2541 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:55:11,063 DEBUG [Thread-2541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:11,063 DEBUG [Thread-2541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:11,063 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:55:11,063 INFO [Thread-2545 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2545. 2024-11-20T16:55:11,064 DEBUG [Thread-2545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:55:11,064 DEBUG [Thread-2545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:11,064 DEBUG [Thread-2545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:11,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:55:11,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:11,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121771136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121771136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:11,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:11,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:11,189 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f14228cc7e644c5390b51e0e6c15fa1b_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f14228cc7e644c5390b51e0e6c15fa1b_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:11,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/ca753a0d87f54611aaad3e00af56a737, store: [table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:11,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/ca753a0d87f54611aaad3e00af56a737 is 175, key is test_row_0/A:col10/1732121709867/Put/seqid=0 2024-11-20T16:55:11,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742476_1652 (size=31255) 2024-11-20T16:55:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-20T16:55:11,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:11,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121771346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121771346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:11,596 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=425, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/ca753a0d87f54611aaad3e00af56a737 2024-11-20T16:55:11,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/820a955f78744126b9be6bde56b84135 is 50, key is test_row_0/B:col10/1732121709867/Put/seqid=0 2024-11-20T16:55:11,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742477_1653 (size=12301) 2024-11-20T16:55:11,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121771656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:11,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121771657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:11,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-20T16:55:12,014 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/820a955f78744126b9be6bde56b84135 2024-11-20T16:55:12,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/2cb6dc5bda564c8b90a337c2cbade06e is 50, key is test_row_0/C:col10/1732121709867/Put/seqid=0 2024-11-20T16:55:12,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742478_1654 (size=12301) 2024-11-20T16:55:12,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:12,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T16:55:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34010 deadline: 1732121772166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33929 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34000 deadline: 1732121772166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:12,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:12,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 , the old value is region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:12,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:12,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T16:55:12,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:12,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., hostname=b60eac9bda43,33929,1732121526384, seqNum=5 because the exception is null or not the one we care about 2024-11-20T16:55:12,431 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/2cb6dc5bda564c8b90a337c2cbade06e 2024-11-20T16:55:12,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/ca753a0d87f54611aaad3e00af56a737 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/ca753a0d87f54611aaad3e00af56a737 2024-11-20T16:55:12,488 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/ca753a0d87f54611aaad3e00af56a737, entries=150, sequenceid=425, filesize=30.5 K 2024-11-20T16:55:12,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/820a955f78744126b9be6bde56b84135 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/820a955f78744126b9be6bde56b84135 2024-11-20T16:55:12,493 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/820a955f78744126b9be6bde56b84135, entries=150, sequenceid=425, filesize=12.0 K 2024-11-20T16:55:12,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/2cb6dc5bda564c8b90a337c2cbade06e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/2cb6dc5bda564c8b90a337c2cbade06e 2024-11-20T16:55:12,497 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/2cb6dc5bda564c8b90a337c2cbade06e, entries=150, sequenceid=425, filesize=12.0 K 2024-11-20T16:55:12,498 INFO [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 7565e0cf39c553ea85172f8e95032446 in 1725ms, sequenceid=425, compaction requested=false 2024-11-20T16:55:12,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:12,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:12,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-11-20T16:55:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-11-20T16:55:12,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-20T16:55:12,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8790 sec 2024-11-20T16:55:12,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 1.8860 sec 2024-11-20T16:55:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-20T16:55:12,766 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:55:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33929 {}] regionserver.HRegion(8855): Flush requested on 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:13,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7565e0cf39c553ea85172f8e95032446 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T16:55:13,180 INFO [Thread-2528 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2528. 2024-11-20T16:55:13,180 INFO [Thread-2534 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2534. 2024-11-20T16:55:13,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=A 2024-11-20T16:55:13,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:55:13,180 DEBUG [Thread-2534 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:55:13,180 DEBUG [Thread-2528 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:55:13,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=B 2024-11-20T16:55:13,180 DEBUG [Thread-2534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:13,180 DEBUG [Thread-2528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:13,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:55:13,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=C 2024-11-20T16:55:13,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:55:13,181 DEBUG [Thread-2534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:13,181 DEBUG [Thread-2528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:13,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:55:13,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:55:13,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cdc435ae3c054b8f9adc59a71ba44c86_7565e0cf39c553ea85172f8e95032446 is 50, key is test_row_0/A:col10/1732121713177/Put/seqid=0 2024-11-20T16:55:13,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742479_1655 (size=12454) 2024-11-20T16:55:13,593 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:13,600 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cdc435ae3c054b8f9adc59a71ba44c86_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cdc435ae3c054b8f9adc59a71ba44c86_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:13,601 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/65ba1090d1ba47559865523f95fcdc27, store: [table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:13,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/65ba1090d1ba47559865523f95fcdc27 is 175, key is test_row_0/A:col10/1732121713177/Put/seqid=0 2024-11-20T16:55:13,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742480_1656 (size=31255) 2024-11-20T16:55:14,008 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=439, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/65ba1090d1ba47559865523f95fcdc27 2024-11-20T16:55:14,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/231e6c94158d455c95c1351fb0d1313f is 50, key is test_row_0/B:col10/1732121713177/Put/seqid=0 2024-11-20T16:55:14,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742481_1657 (size=12301) 2024-11-20T16:55:14,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/231e6c94158d455c95c1351fb0d1313f 2024-11-20T16:55:14,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/400b9b184b754d72ab67d75299f44e07 is 50, key is test_row_0/C:col10/1732121713177/Put/seqid=0 2024-11-20T16:55:14,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742482_1658 (size=12301) 2024-11-20T16:55:14,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/400b9b184b754d72ab67d75299f44e07 2024-11-20T16:55:14,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/65ba1090d1ba47559865523f95fcdc27 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/65ba1090d1ba47559865523f95fcdc27 2024-11-20T16:55:14,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/65ba1090d1ba47559865523f95fcdc27, entries=150, sequenceid=439, filesize=30.5 K 2024-11-20T16:55:14,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/231e6c94158d455c95c1351fb0d1313f as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/231e6c94158d455c95c1351fb0d1313f 2024-11-20T16:55:14,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/231e6c94158d455c95c1351fb0d1313f, entries=150, sequenceid=439, filesize=12.0 K 2024-11-20T16:55:14,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/400b9b184b754d72ab67d75299f44e07 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/400b9b184b754d72ab67d75299f44e07 2024-11-20T16:55:14,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/400b9b184b754d72ab67d75299f44e07, entries=150, sequenceid=439, filesize=12.0 K 2024-11-20T16:55:14,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 7565e0cf39c553ea85172f8e95032446 in 1685ms, sequenceid=439, compaction requested=true 2024-11-20T16:55:14,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:14,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T16:55:14,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:14,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T16:55:14,864 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:55:14,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:14,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7565e0cf39c553ea85172f8e95032446:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T16:55:14,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:55:14,864 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:55:14,865 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:55:14,865 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:55:14,865 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/A is initiating minor compaction (all files) 2024-11-20T16:55:14,865 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/B is initiating minor compaction (all files) 2024-11-20T16:55:14,865 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/A in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:14,865 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/B in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:14,865 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/12913cd139fb45bb8fe0e776322848bf, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/ca753a0d87f54611aaad3e00af56a737, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/65ba1090d1ba47559865523f95fcdc27] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=92.2 K 2024-11-20T16:55:14,865 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/0280f43e249e429b8d5634cbea52b7d8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/820a955f78744126b9be6bde56b84135, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/231e6c94158d455c95c1351fb0d1313f] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=36.7 K 2024-11-20T16:55:14,865 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:14,865 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. files: [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/12913cd139fb45bb8fe0e776322848bf, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/ca753a0d87f54611aaad3e00af56a737, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/65ba1090d1ba47559865523f95fcdc27] 2024-11-20T16:55:14,865 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0280f43e249e429b8d5634cbea52b7d8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732121709749 2024-11-20T16:55:14,865 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 12913cd139fb45bb8fe0e776322848bf, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732121709749 2024-11-20T16:55:14,865 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 820a955f78744126b9be6bde56b84135, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732121709860 2024-11-20T16:55:14,865 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting ca753a0d87f54611aaad3e00af56a737, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732121709860 2024-11-20T16:55:14,865 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 231e6c94158d455c95c1351fb0d1313f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732121711023 2024-11-20T16:55:14,865 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] compactions.Compactor(225): Compacting 65ba1090d1ba47559865523f95fcdc27, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732121711023 2024-11-20T16:55:14,870 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:14,872 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#B#compaction#547 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:55:14,872 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112073761b33ad254c21b0d369f477b2389c_7565e0cf39c553ea85172f8e95032446 store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:14,872 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/519e84722f824f14bbd728ec0992c630 is 50, key is test_row_0/B:col10/1732121713177/Put/seqid=0 2024-11-20T16:55:14,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742483_1659 (size=13051) 2024-11-20T16:55:14,876 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112073761b33ad254c21b0d369f477b2389c_7565e0cf39c553ea85172f8e95032446, store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:14,876 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112073761b33ad254c21b0d369f477b2389c_7565e0cf39c553ea85172f8e95032446 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:14,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742484_1660 (size=4469) 2024-11-20T16:55:15,282 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#A#compaction#548 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:55:15,283 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/69564697fbd9420f8ea556d599136a5d is 175, key is test_row_0/A:col10/1732121713177/Put/seqid=0 2024-11-20T16:55:15,287 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/519e84722f824f14bbd728ec0992c630 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/519e84722f824f14bbd728ec0992c630 2024-11-20T16:55:15,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742485_1661 (size=32005) 2024-11-20T16:55:15,292 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/B of 7565e0cf39c553ea85172f8e95032446 into 519e84722f824f14bbd728ec0992c630(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:55:15,292 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:15,292 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/B, priority=13, startTime=1732121714864; duration=0sec 2024-11-20T16:55:15,292 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T16:55:15,292 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:B 2024-11-20T16:55:15,292 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T16:55:15,293 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T16:55:15,293 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1541): 7565e0cf39c553ea85172f8e95032446/C is initiating minor compaction (all files) 2024-11-20T16:55:15,293 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7565e0cf39c553ea85172f8e95032446/C in TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:15,294 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/b1ad589f6b3946058f2b568c1afbc2ab, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/2cb6dc5bda564c8b90a337c2cbade06e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/400b9b184b754d72ab67d75299f44e07] into tmpdir=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp, totalSize=36.7 K 2024-11-20T16:55:15,294 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1ad589f6b3946058f2b568c1afbc2ab, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732121709749 2024-11-20T16:55:15,294 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cb6dc5bda564c8b90a337c2cbade06e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732121709860 2024-11-20T16:55:15,295 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 400b9b184b754d72ab67d75299f44e07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732121711023 2024-11-20T16:55:15,302 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7565e0cf39c553ea85172f8e95032446#C#compaction#549 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T16:55:15,302 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/39bc7fc223654131b1a0177e97e3853b is 50, key is test_row_0/C:col10/1732121713177/Put/seqid=0 2024-11-20T16:55:15,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742486_1662 (size=13051) 2024-11-20T16:55:15,696 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/69564697fbd9420f8ea556d599136a5d as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/69564697fbd9420f8ea556d599136a5d 2024-11-20T16:55:15,704 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/A of 7565e0cf39c553ea85172f8e95032446 into 69564697fbd9420f8ea556d599136a5d(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:55:15,704 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:15,704 INFO [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/A, priority=13, startTime=1732121714864; duration=0sec 2024-11-20T16:55:15,704 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:15,704 DEBUG [RS:0;b60eac9bda43:33929-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:A 2024-11-20T16:55:15,712 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/39bc7fc223654131b1a0177e97e3853b as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/39bc7fc223654131b1a0177e97e3853b 2024-11-20T16:55:15,718 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7565e0cf39c553ea85172f8e95032446/C of 7565e0cf39c553ea85172f8e95032446 into 39bc7fc223654131b1a0177e97e3853b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T16:55:15,718 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7565e0cf39c553ea85172f8e95032446: 2024-11-20T16:55:15,718 INFO [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., storeName=7565e0cf39c553ea85172f8e95032446/C, priority=13, startTime=1732121714864; duration=0sec 2024-11-20T16:55:15,718 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T16:55:15,718 DEBUG [RS:0;b60eac9bda43:33929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7565e0cf39c553ea85172f8e95032446:C 2024-11-20T16:55:20,530 INFO [Thread-2530 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2530. 2024-11-20T16:55:20,530 DEBUG [Thread-2530 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:55:20,530 DEBUG [Thread-2530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:20,530 DEBUG [Thread-2530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:20,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:55:20,547 INFO [Thread-2536 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2536. 2024-11-20T16:55:20,547 DEBUG [Thread-2536 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:55:20,547 DEBUG [Thread-2536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:20,547 DEBUG [Thread-2536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:20,547 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:55:20,578 INFO [Thread-2532 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2532. 2024-11-20T16:55:20,578 DEBUG [Thread-2532 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T16:55:20,578 DEBUG [Thread-2532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:20,578 DEBUG [Thread-2532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:20,578 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:55:20,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T16:55:20,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 152 2024-11-20T16:55:20,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-20T16:55:20,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-20T16:55:20,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 183 2024-11-20T16:55:20,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-20T16:55:20,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T16:55:20,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3183 2024-11-20T16:55:20,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3057 2024-11-20T16:55:20,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3210 2024-11-20T16:55:20,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3188 2024-11-20T16:55:20,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3056 2024-11-20T16:55:20,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T16:55:20,579 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T16:55:20,579 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T16:55:20,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:20,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:20,580 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:55:20,581 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T16:55:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T16:55:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-20T16:55:20,586 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121720586"}]},"ts":"1732121720586"} 2024-11-20T16:55:20,589 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T16:55:20,589 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T16:55:20,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T16:55:20,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7565e0cf39c553ea85172f8e95032446, UNASSIGN}] 2024-11-20T16:55:20,593 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7565e0cf39c553ea85172f8e95032446, UNASSIGN 2024-11-20T16:55:20,593 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=178 updating hbase:meta row=7565e0cf39c553ea85172f8e95032446, regionState=CLOSING, regionLocation=b60eac9bda43,33929,1732121526384 2024-11-20T16:55:20,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7565e0cf39c553ea85172f8e95032446, UNASSIGN because future has completed 2024-11-20T16:55:20,595 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T16:55:20,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384}] 2024-11-20T16:55:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-20T16:55:20,747 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(122): Close 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:20,747 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T16:55:20,747 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1722): Closing 7565e0cf39c553ea85172f8e95032446, disabling compactions & flushes 2024-11-20T16:55:20,747 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:20,747 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:20,747 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. after waiting 0 ms 2024-11-20T16:55:20,747 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:20,748 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(2902): Flushing 7565e0cf39c553ea85172f8e95032446 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-20T16:55:20,748 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=A 2024-11-20T16:55:20,748 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:55:20,748 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=B 2024-11-20T16:55:20,748 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:55:20,748 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7565e0cf39c553ea85172f8e95032446, store=C 2024-11-20T16:55:20,748 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T16:55:20,752 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208e04a7bab6614e2e91d23eccccf75dc4_7565e0cf39c553ea85172f8e95032446 is 50, key is test_row_1/A:col10/1732121720576/Put/seqid=0 2024-11-20T16:55:20,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742487_1663 (size=9914) 2024-11-20T16:55:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-20T16:55:21,157 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T16:55:21,165 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208e04a7bab6614e2e91d23eccccf75dc4_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208e04a7bab6614e2e91d23eccccf75dc4_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:21,167 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/72676a3fb77a4c509922b987d1d24c2a, store: [table=TestAcidGuarantees family=A region=7565e0cf39c553ea85172f8e95032446] 2024-11-20T16:55:21,169 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/72676a3fb77a4c509922b987d1d24c2a is 175, key is test_row_1/A:col10/1732121720576/Put/seqid=0 2024-11-20T16:55:21,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742488_1664 (size=22561) 2024-11-20T16:55:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-20T16:55:21,573 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=448, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/72676a3fb77a4c509922b987d1d24c2a 2024-11-20T16:55:21,578 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/5b1c4c8bc09646959ea93d50ad7a16dc is 50, key is test_row_1/B:col10/1732121720576/Put/seqid=0 2024-11-20T16:55:21,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742489_1665 (size=9857) 2024-11-20T16:55:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-20T16:55:21,982 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/5b1c4c8bc09646959ea93d50ad7a16dc 2024-11-20T16:55:21,994 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/4ca56377ca754667a477a511e28b5b83 is 50, key is test_row_1/C:col10/1732121720576/Put/seqid=0 2024-11-20T16:55:21,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742490_1666 (size=9857) 2024-11-20T16:55:22,399 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/4ca56377ca754667a477a511e28b5b83 2024-11-20T16:55:22,408 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/A/72676a3fb77a4c509922b987d1d24c2a as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/72676a3fb77a4c509922b987d1d24c2a 2024-11-20T16:55:22,412 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/72676a3fb77a4c509922b987d1d24c2a, entries=100, sequenceid=448, filesize=22.0 K 2024-11-20T16:55:22,413 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/B/5b1c4c8bc09646959ea93d50ad7a16dc as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/5b1c4c8bc09646959ea93d50ad7a16dc 2024-11-20T16:55:22,416 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/5b1c4c8bc09646959ea93d50ad7a16dc, entries=100, sequenceid=448, filesize=9.6 K 2024-11-20T16:55:22,416 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/.tmp/C/4ca56377ca754667a477a511e28b5b83 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/4ca56377ca754667a477a511e28b5b83 2024-11-20T16:55:22,419 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/4ca56377ca754667a477a511e28b5b83, entries=100, sequenceid=448, filesize=9.6 K 2024-11-20T16:55:22,420 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 7565e0cf39c553ea85172f8e95032446 in 1673ms, sequenceid=448, compaction requested=false 2024-11-20T16:55:22,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/f0bf7017aa554852b7f731d79bc13b18, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/2c65775d4f7c4d8e93d09621c2401562, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/37b05625a689466f832dc1e3dbdb1688, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/0e15d72d64bb4d8b80bd6ff7c74c766e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/75b1a01674eb4a359892a3f01a936b21, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/a42433f197394c97845071bd43463654, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/6761962959eb4345ac7c2d4a98ea2e65, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/db5791b06bed467dbbf2021707e08014, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/0e01444c6fa640ce952fac3da3caef55, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/68680c4598ea478e8c8e129c000d6e6b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/162741c6de7446c78f04b97a6244ab08, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/57e1ea4f913c4db1b87fbb164b7f5a14, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/b7809fabdd534e55ba678476a751c625, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/d3035ebb1c204ae0875ea2bcb69313f8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/9ecf5d1ff1f9415595541b52142e523f, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/62d5faf4dda549e0b291b74c5ab1e1b4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/058873af73214715a57b789f963e94ca, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/eeeaf6b2c28c4642ba58a483875b0d1a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/12913cd139fb45bb8fe0e776322848bf, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/4ff11cb613da4209a0f6bed099df3c49, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/ca753a0d87f54611aaad3e00af56a737, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/65ba1090d1ba47559865523f95fcdc27] to archive 2024-11-20T16:55:22,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T16:55:22,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/f0bf7017aa554852b7f731d79bc13b18 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/f0bf7017aa554852b7f731d79bc13b18 2024-11-20T16:55:22,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/2c65775d4f7c4d8e93d09621c2401562 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/2c65775d4f7c4d8e93d09621c2401562 2024-11-20T16:55:22,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/37b05625a689466f832dc1e3dbdb1688 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/37b05625a689466f832dc1e3dbdb1688 2024-11-20T16:55:22,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/0e15d72d64bb4d8b80bd6ff7c74c766e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/0e15d72d64bb4d8b80bd6ff7c74c766e 2024-11-20T16:55:22,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/75b1a01674eb4a359892a3f01a936b21 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/75b1a01674eb4a359892a3f01a936b21 2024-11-20T16:55:22,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/a42433f197394c97845071bd43463654 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/a42433f197394c97845071bd43463654 2024-11-20T16:55:22,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/6761962959eb4345ac7c2d4a98ea2e65 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/6761962959eb4345ac7c2d4a98ea2e65 2024-11-20T16:55:22,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/db5791b06bed467dbbf2021707e08014 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/db5791b06bed467dbbf2021707e08014 2024-11-20T16:55:22,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/0e01444c6fa640ce952fac3da3caef55 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/0e01444c6fa640ce952fac3da3caef55 2024-11-20T16:55:22,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/68680c4598ea478e8c8e129c000d6e6b to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/68680c4598ea478e8c8e129c000d6e6b 2024-11-20T16:55:22,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/162741c6de7446c78f04b97a6244ab08 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/162741c6de7446c78f04b97a6244ab08 2024-11-20T16:55:22,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/57e1ea4f913c4db1b87fbb164b7f5a14 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/57e1ea4f913c4db1b87fbb164b7f5a14 2024-11-20T16:55:22,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/b7809fabdd534e55ba678476a751c625 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/b7809fabdd534e55ba678476a751c625 2024-11-20T16:55:22,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/d3035ebb1c204ae0875ea2bcb69313f8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/d3035ebb1c204ae0875ea2bcb69313f8 2024-11-20T16:55:22,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/9ecf5d1ff1f9415595541b52142e523f to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/9ecf5d1ff1f9415595541b52142e523f 2024-11-20T16:55:22,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/62d5faf4dda549e0b291b74c5ab1e1b4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/62d5faf4dda549e0b291b74c5ab1e1b4 2024-11-20T16:55:22,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/058873af73214715a57b789f963e94ca to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/058873af73214715a57b789f963e94ca 2024-11-20T16:55:22,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/eeeaf6b2c28c4642ba58a483875b0d1a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/eeeaf6b2c28c4642ba58a483875b0d1a 2024-11-20T16:55:22,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/12913cd139fb45bb8fe0e776322848bf to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/12913cd139fb45bb8fe0e776322848bf 2024-11-20T16:55:22,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/4ff11cb613da4209a0f6bed099df3c49 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/4ff11cb613da4209a0f6bed099df3c49 2024-11-20T16:55:22,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/ca753a0d87f54611aaad3e00af56a737 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/ca753a0d87f54611aaad3e00af56a737 2024-11-20T16:55:22,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/65ba1090d1ba47559865523f95fcdc27 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/65ba1090d1ba47559865523f95fcdc27 2024-11-20T16:55:22,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/dffe009e180547a2a66f63c5e5ce8dde, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/4a2dfa99b0804ff89101673641f4af7c, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/072c727a69964d6085cf40ca2233fb16, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/3ed74b5b571a459f9926e249b4c77585, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/c8ae4612c1dc49d2a531aa63704db985, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/69689ebba94740f4821866662188f6b0, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/c9620f0c727e47ebab4b65c8372a3ba1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/8023a131e1b94940aec9d01107298939, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/f2b7904e49e74172a5ce18e473d1d5e1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/25af895a111b4473942a8b2dfe377e68, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/184c8debf00545a98398368a4c9b3242, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/4322748d7bff4adc8a9f3ab9a121b9c5, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/22610380ced14960b76beaae7e3ba39d, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/4c4016c0625c486897a28297948a9c05, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/8dc8287bcdcf4dd6baafec240b355859, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/6d9f6d44134b4718b08b899c577db7b7, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/e8f977d25a664c9fb0c9ff26b5d39524, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/b5bc1429ba28436ca991e333a4e3804b, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/0280f43e249e429b8d5634cbea52b7d8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/fddee825f45c4ea3b3937ef65dacf11e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/820a955f78744126b9be6bde56b84135, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/231e6c94158d455c95c1351fb0d1313f] to archive 2024-11-20T16:55:22,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T16:55:22,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/dffe009e180547a2a66f63c5e5ce8dde to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/dffe009e180547a2a66f63c5e5ce8dde 2024-11-20T16:55:22,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/4a2dfa99b0804ff89101673641f4af7c to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/4a2dfa99b0804ff89101673641f4af7c 2024-11-20T16:55:22,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/072c727a69964d6085cf40ca2233fb16 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/072c727a69964d6085cf40ca2233fb16 2024-11-20T16:55:22,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/3ed74b5b571a459f9926e249b4c77585 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/3ed74b5b571a459f9926e249b4c77585 2024-11-20T16:55:22,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/c8ae4612c1dc49d2a531aa63704db985 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/c8ae4612c1dc49d2a531aa63704db985 2024-11-20T16:55:22,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/69689ebba94740f4821866662188f6b0 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/69689ebba94740f4821866662188f6b0 2024-11-20T16:55:22,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/c9620f0c727e47ebab4b65c8372a3ba1 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/c9620f0c727e47ebab4b65c8372a3ba1 2024-11-20T16:55:22,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/8023a131e1b94940aec9d01107298939 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/8023a131e1b94940aec9d01107298939 2024-11-20T16:55:22,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/f2b7904e49e74172a5ce18e473d1d5e1 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/f2b7904e49e74172a5ce18e473d1d5e1 2024-11-20T16:55:22,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/25af895a111b4473942a8b2dfe377e68 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/25af895a111b4473942a8b2dfe377e68 2024-11-20T16:55:22,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/184c8debf00545a98398368a4c9b3242 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/184c8debf00545a98398368a4c9b3242 2024-11-20T16:55:22,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/4322748d7bff4adc8a9f3ab9a121b9c5 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/4322748d7bff4adc8a9f3ab9a121b9c5 2024-11-20T16:55:22,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/22610380ced14960b76beaae7e3ba39d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/22610380ced14960b76beaae7e3ba39d 2024-11-20T16:55:22,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/4c4016c0625c486897a28297948a9c05 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/4c4016c0625c486897a28297948a9c05 2024-11-20T16:55:22,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/8dc8287bcdcf4dd6baafec240b355859 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/8dc8287bcdcf4dd6baafec240b355859 2024-11-20T16:55:22,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/6d9f6d44134b4718b08b899c577db7b7 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/6d9f6d44134b4718b08b899c577db7b7 2024-11-20T16:55:22,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/e8f977d25a664c9fb0c9ff26b5d39524 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/e8f977d25a664c9fb0c9ff26b5d39524 2024-11-20T16:55:22,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/b5bc1429ba28436ca991e333a4e3804b to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/b5bc1429ba28436ca991e333a4e3804b 2024-11-20T16:55:22,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/0280f43e249e429b8d5634cbea52b7d8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/0280f43e249e429b8d5634cbea52b7d8 2024-11-20T16:55:22,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/fddee825f45c4ea3b3937ef65dacf11e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/fddee825f45c4ea3b3937ef65dacf11e 2024-11-20T16:55:22,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/820a955f78744126b9be6bde56b84135 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/820a955f78744126b9be6bde56b84135 2024-11-20T16:55:22,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/231e6c94158d455c95c1351fb0d1313f to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/231e6c94158d455c95c1351fb0d1313f 2024-11-20T16:55:22,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/0d10e202e7ea419e86b5071d3eb28da4, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/4f1c6dbe8e0b4e35a8585a454d784d79, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/56c666c554a746c19b499e29131ebf34, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/1297a6cfe7bd4261a4eccc8067e02ff1, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/92b138868ef54d76a2eab3da87464bd9, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/81b8ef84a8ef4250b222eb3c50eb3273, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/c347d37165b34d97a5a5bb8545d91995, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/b22430499496456c96358548badafd51, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/7cbf4a9ef4e3441497466b70d487a04a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/f3ce8405f1ac4dfab8cba233a375052a, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/9a9c79ec42ad472387ecf463fb0d6e1e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/ba89e77f0677497885471d8a8284fa13, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/81bf03b6135849cdb1e9b8648f075607, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/29b5443c9a6244be84cffc62e28cea15, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/0a90726202ad4f90a604e46e26b3d302, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/f1ab919755ec4a019eac8fc10deeeea8, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/f2fc0fb6ccdf496195ac44ea8dcda5dd, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/6716f130c6ac479da13ab5993c0da67e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/b1ad589f6b3946058f2b568c1afbc2ab, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/07960884c4e94e269e7d8e914464bb74, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/2cb6dc5bda564c8b90a337c2cbade06e, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/400b9b184b754d72ab67d75299f44e07] to archive 2024-11-20T16:55:22,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T16:55:22,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/0d10e202e7ea419e86b5071d3eb28da4 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/0d10e202e7ea419e86b5071d3eb28da4 2024-11-20T16:55:22,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/4f1c6dbe8e0b4e35a8585a454d784d79 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/4f1c6dbe8e0b4e35a8585a454d784d79 2024-11-20T16:55:22,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/56c666c554a746c19b499e29131ebf34 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/56c666c554a746c19b499e29131ebf34 2024-11-20T16:55:22,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/1297a6cfe7bd4261a4eccc8067e02ff1 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/1297a6cfe7bd4261a4eccc8067e02ff1 2024-11-20T16:55:22,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/92b138868ef54d76a2eab3da87464bd9 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/92b138868ef54d76a2eab3da87464bd9 2024-11-20T16:55:22,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/81b8ef84a8ef4250b222eb3c50eb3273 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/81b8ef84a8ef4250b222eb3c50eb3273 2024-11-20T16:55:22,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/c347d37165b34d97a5a5bb8545d91995 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/c347d37165b34d97a5a5bb8545d91995 2024-11-20T16:55:22,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/b22430499496456c96358548badafd51 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/b22430499496456c96358548badafd51 2024-11-20T16:55:22,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/7cbf4a9ef4e3441497466b70d487a04a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/7cbf4a9ef4e3441497466b70d487a04a 2024-11-20T16:55:22,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/f3ce8405f1ac4dfab8cba233a375052a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/f3ce8405f1ac4dfab8cba233a375052a 2024-11-20T16:55:22,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/9a9c79ec42ad472387ecf463fb0d6e1e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/9a9c79ec42ad472387ecf463fb0d6e1e 2024-11-20T16:55:22,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/ba89e77f0677497885471d8a8284fa13 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/ba89e77f0677497885471d8a8284fa13 2024-11-20T16:55:22,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/81bf03b6135849cdb1e9b8648f075607 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/81bf03b6135849cdb1e9b8648f075607 2024-11-20T16:55:22,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/29b5443c9a6244be84cffc62e28cea15 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/29b5443c9a6244be84cffc62e28cea15 2024-11-20T16:55:22,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/0a90726202ad4f90a604e46e26b3d302 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/0a90726202ad4f90a604e46e26b3d302 2024-11-20T16:55:22,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/f1ab919755ec4a019eac8fc10deeeea8 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/f1ab919755ec4a019eac8fc10deeeea8 2024-11-20T16:55:22,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/f2fc0fb6ccdf496195ac44ea8dcda5dd to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/f2fc0fb6ccdf496195ac44ea8dcda5dd 2024-11-20T16:55:22,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/6716f130c6ac479da13ab5993c0da67e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/6716f130c6ac479da13ab5993c0da67e 2024-11-20T16:55:22,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/b1ad589f6b3946058f2b568c1afbc2ab to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/b1ad589f6b3946058f2b568c1afbc2ab 2024-11-20T16:55:22,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/07960884c4e94e269e7d8e914464bb74 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/07960884c4e94e269e7d8e914464bb74 2024-11-20T16:55:22,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/2cb6dc5bda564c8b90a337c2cbade06e to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/2cb6dc5bda564c8b90a337c2cbade06e 2024-11-20T16:55:22,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/400b9b184b754d72ab67d75299f44e07 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/400b9b184b754d72ab67d75299f44e07 2024-11-20T16:55:22,472 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/recovered.edits/451.seqid, newMaxSeqId=451, maxSeqId=4 2024-11-20T16:55:22,472 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. 2024-11-20T16:55:22,472 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1676): Region close journal for 7565e0cf39c553ea85172f8e95032446: Waiting for close lock at 1732121720747Running coprocessor pre-close hooks at 1732121720747Disabling compacts and flushes for region at 1732121720747Disabling writes for close at 1732121720747Obtaining lock to block concurrent updates at 1732121720748 (+1 ms)Preparing flush snapshotting stores in 7565e0cf39c553ea85172f8e95032446 at 1732121720748Finished memstore snapshotting TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732121720748Flushing stores of TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446. at 1732121720748Flushing 7565e0cf39c553ea85172f8e95032446/A: creating writer at 1732121720748Flushing mob file 7565e0cf39c553ea85172f8e95032446/A: appending metadata at 1732121720752 (+4 ms)Flushing mob file 7565e0cf39c553ea85172f8e95032446/A: closing flushed file at 1732121720752Flushing 7565e0cf39c553ea85172f8e95032446/A: appending metadata at 1732121721167 (+415 ms)Flushing 7565e0cf39c553ea85172f8e95032446/A: closing flushed file at 1732121721167Flushing 7565e0cf39c553ea85172f8e95032446/B: creating writer at 1732121721576 (+409 ms)Flushing 7565e0cf39c553ea85172f8e95032446/B: appending metadata at 1732121721578 (+2 ms)Flushing 7565e0cf39c553ea85172f8e95032446/B: closing flushed file at 1732121721578Flushing 7565e0cf39c553ea85172f8e95032446/C: creating writer at 1732121721991 (+413 ms)Flushing 7565e0cf39c553ea85172f8e95032446/C: appending metadata at 1732121721994 (+3 ms)Flushing 7565e0cf39c553ea85172f8e95032446/C: closing flushed file at 1732121721994Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@242e85e2: reopening flushed file at 1732121722406 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4dcaf722: reopening flushed file at 1732121722412 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d735bc0: reopening flushed file at 1732121722416 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 7565e0cf39c553ea85172f8e95032446 in 1673ms, sequenceid=448, compaction requested=false at 1732121722420 (+4 ms)Writing region close event to WAL at 1732121722469 (+49 ms)Running coprocessor post-close hooks at 1732121722472 (+3 ms)Closed at 1732121722472 2024-11-20T16:55:22,474 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(157): Closed 7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,475 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=178 updating hbase:meta row=7565e0cf39c553ea85172f8e95032446, regionState=CLOSED 2024-11-20T16:55:22,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=179, ppid=178, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 because future has completed 2024-11-20T16:55:22,478 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-11-20T16:55:22,478 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; CloseRegionProcedure 7565e0cf39c553ea85172f8e95032446, server=b60eac9bda43,33929,1732121526384 in 1.8810 sec 2024-11-20T16:55:22,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=178, resume processing ppid=177 2024-11-20T16:55:22,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, ppid=177, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7565e0cf39c553ea85172f8e95032446, UNASSIGN in 1.8860 sec 2024-11-20T16:55:22,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-20T16:55:22,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8890 sec 2024-11-20T16:55:22,481 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732121722481"}]},"ts":"1732121722481"} 2024-11-20T16:55:22,482 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T16:55:22,482 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T16:55:22,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9020 sec 2024-11-20T16:55:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-20T16:55:22,737 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:55:22,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T16:55:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:55:22,742 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=180, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:55:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-20T16:55:22,744 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=180, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:55:22,749 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,752 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A, FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B, FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C, FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/recovered.edits] 2024-11-20T16:55:22,756 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/69564697fbd9420f8ea556d599136a5d to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/69564697fbd9420f8ea556d599136a5d 2024-11-20T16:55:22,757 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/72676a3fb77a4c509922b987d1d24c2a to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/A/72676a3fb77a4c509922b987d1d24c2a 2024-11-20T16:55:22,761 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/519e84722f824f14bbd728ec0992c630 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/519e84722f824f14bbd728ec0992c630 2024-11-20T16:55:22,763 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/5b1c4c8bc09646959ea93d50ad7a16dc to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/B/5b1c4c8bc09646959ea93d50ad7a16dc 2024-11-20T16:55:22,767 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/39bc7fc223654131b1a0177e97e3853b to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/39bc7fc223654131b1a0177e97e3853b 2024-11-20T16:55:22,769 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/4ca56377ca754667a477a511e28b5b83 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/C/4ca56377ca754667a477a511e28b5b83 2024-11-20T16:55:22,773 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/recovered.edits/451.seqid to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446/recovered.edits/451.seqid 2024-11-20T16:55:22,774 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/default/TestAcidGuarantees/7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,774 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T16:55:22,775 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T16:55:22,776 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T16:55:22,781 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112000abf9e144484219ace637fb6d108ee1_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112000abf9e144484219ace637fb6d108ee1_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,783 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120068ea9cbe4374d8d8607c2e6ec957864_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120068ea9cbe4374d8d8607c2e6ec957864_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,784 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112008fd6ac24516416281226cf5983577ce_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112008fd6ac24516416281226cf5983577ce_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,786 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120416afed65cb14468b67b6789c3602faf_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120416afed65cb14468b67b6789c3602faf_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,788 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112057ee5b59d6c044c093db0fc547b58b4b_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112057ee5b59d6c044c093db0fc547b58b4b_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,789 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205d3ad214f0f84593a66efa25543bfb7e_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205d3ad214f0f84593a66efa25543bfb7e_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,791 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120635186c4a5d94a7e87be3f0131d0367a_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120635186c4a5d94a7e87be3f0131d0367a_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,793 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208e04a7bab6614e2e91d23eccccf75dc4_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208e04a7bab6614e2e91d23eccccf75dc4_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,794 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209660b6e36c734243a7dbc61672f2633e_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209660b6e36c734243a7dbc61672f2633e_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,795 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a1f9e23588274520bb805b41e07b4ba3_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a1f9e23588274520bb805b41e07b4ba3_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,797 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120af6be5fbc4c34993b1cbe2e849b7626c_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120af6be5fbc4c34993b1cbe2e849b7626c_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,798 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b5b799dce4194e55a55b23e5b31a01a1_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b5b799dce4194e55a55b23e5b31a01a1_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,800 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120be12502514fe46178fb5d3e9e6e8c65a_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120be12502514fe46178fb5d3e9e6e8c65a_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,801 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c34b36d1fe4144259ef19be31c8a25c8_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c34b36d1fe4144259ef19be31c8a25c8_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,803 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cdc435ae3c054b8f9adc59a71ba44c86_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cdc435ae3c054b8f9adc59a71ba44c86_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,804 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ec8b70c39a574ad8b9bf8a144f79e526_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ec8b70c39a574ad8b9bf8a144f79e526_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,805 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f14228cc7e644c5390b51e0e6c15fa1b_7565e0cf39c553ea85172f8e95032446 to hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f14228cc7e644c5390b51e0e6c15fa1b_7565e0cf39c553ea85172f8e95032446 2024-11-20T16:55:22,806 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T16:55:22,809 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=180, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:55:22,812 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T16:55:22,814 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T16:55:22,815 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=180, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:55:22,815 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T16:55:22,815 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732121722815"}]},"ts":"9223372036854775807"} 2024-11-20T16:55:22,817 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T16:55:22,817 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7565e0cf39c553ea85172f8e95032446, NAME => 'TestAcidGuarantees,,1732121687902.7565e0cf39c553ea85172f8e95032446.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T16:55:22,817 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T16:55:22,817 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732121722817"}]},"ts":"9223372036854775807"} 2024-11-20T16:55:22,818 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T16:55:22,819 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=180, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T16:55:22,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 81 msec 2024-11-20T16:55:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-20T16:55:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T16:55:22,857 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T16:55:22,870 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 247), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=363 (was 462), ProcessCount=11 (was 11), AvailableMemoryMB=7084 (was 7265) 2024-11-20T16:55:22,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T16:55:22,870 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T16:55:22,870 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T16:55:22,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:22,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:22,870 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T16:55:22,870 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T16:55:22,870 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=841754823, stopped=false 2024-11-20T16:55:22,871 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b60eac9bda43,33377,1732121525625 2024-11-20T16:55:22,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T16:55:22,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T16:55:22,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T16:55:22,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T16:55:22,904 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T16:55:22,905 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T16:55:22,905 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T16:55:22,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:22,905 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T16:55:22,905 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T16:55:22,906 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b60eac9bda43,33929,1732121526384' ***** 2024-11-20T16:55:22,906 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T16:55:22,906 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T16:55:22,907 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T16:55:22,907 INFO [RS:0;b60eac9bda43:33929 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T16:55:22,907 INFO [RS:0;b60eac9bda43:33929 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T16:55:22,907 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(959): stopping server b60eac9bda43,33929,1732121526384 2024-11-20T16:55:22,907 INFO [RS:0;b60eac9bda43:33929 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T16:55:22,907 INFO [RS:0;b60eac9bda43:33929 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b60eac9bda43:33929. 2024-11-20T16:55:22,907 DEBUG [RS:0;b60eac9bda43:33929 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T16:55:22,907 DEBUG [RS:0;b60eac9bda43:33929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:22,908 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T16:55:22,908 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T16:55:22,908 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T16:55:22,908 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T16:55:22,909 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T16:55:22,909 DEBUG [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-20T16:55:22,909 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T16:55:22,909 INFO [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T16:55:22,909 DEBUG [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T16:55:22,909 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T16:55:22,909 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T16:55:22,909 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T16:55:22,909 INFO [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-20T16:55:22,935 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/.tmp/info/2cd514f3c15a4cd4a9d9234128268129 is 95, key is TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2./info:/1732121558954/DeleteFamily/seqid=0 2024-11-20T16:55:22,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742491_1667 (size=6514) 2024-11-20T16:55:22,980 INFO [regionserver/b60eac9bda43:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T16:55:23,109 DEBUG [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T16:55:23,310 DEBUG [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T16:55:23,320 INFO [regionserver/b60eac9bda43:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T16:55:23,320 INFO [regionserver/b60eac9bda43:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T16:55:23,339 INFO [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/.tmp/info/2cd514f3c15a4cd4a9d9234128268129 2024-11-20T16:55:23,359 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/.tmp/ns/95b7ef5dba4e4389be1cc0ff5f688097 is 93, key is TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2./ns:/1732121558916/DeleteFamily/seqid=0 2024-11-20T16:55:23,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742492_1668 (size=5906) 2024-11-20T16:55:23,510 DEBUG [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T16:55:23,710 DEBUG [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T16:55:23,763 INFO [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/.tmp/ns/95b7ef5dba4e4389be1cc0ff5f688097 2024-11-20T16:55:23,788 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/.tmp/rep_barrier/39596c8113804337884f395fa37c9b77 is 102, key is TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2./rep_barrier:/1732121558916/DeleteFamily/seqid=0 2024-11-20T16:55:23,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742493_1669 (size=6025) 2024-11-20T16:55:23,911 DEBUG [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T16:55:24,111 DEBUG [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T16:55:24,192 INFO [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/.tmp/rep_barrier/39596c8113804337884f395fa37c9b77 2024-11-20T16:55:24,218 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/.tmp/table/891a3a3fc3474f94b4be7bec051b0643 is 96, key is TestAcidGuarantees,,1732121530990.b683f7af58301e87dbf6f3d58dd983d2./table:/1732121558916/DeleteFamily/seqid=0 2024-11-20T16:55:24,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742494_1670 (size=5936) 2024-11-20T16:55:24,311 DEBUG [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T16:55:24,511 DEBUG [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T16:55:24,622 INFO [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/.tmp/table/891a3a3fc3474f94b4be7bec051b0643 2024-11-20T16:55:24,632 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/.tmp/info/2cd514f3c15a4cd4a9d9234128268129 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/info/2cd514f3c15a4cd4a9d9234128268129 2024-11-20T16:55:24,639 INFO [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/info/2cd514f3c15a4cd4a9d9234128268129, entries=12, sequenceid=90, filesize=6.4 K 2024-11-20T16:55:24,641 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/.tmp/ns/95b7ef5dba4e4389be1cc0ff5f688097 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/ns/95b7ef5dba4e4389be1cc0ff5f688097 2024-11-20T16:55:24,644 INFO [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/ns/95b7ef5dba4e4389be1cc0ff5f688097, entries=8, sequenceid=90, filesize=5.8 K 2024-11-20T16:55:24,645 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/.tmp/rep_barrier/39596c8113804337884f395fa37c9b77 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/rep_barrier/39596c8113804337884f395fa37c9b77 2024-11-20T16:55:24,647 INFO [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/rep_barrier/39596c8113804337884f395fa37c9b77, entries=6, sequenceid=90, filesize=5.9 K 2024-11-20T16:55:24,648 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/.tmp/table/891a3a3fc3474f94b4be7bec051b0643 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/table/891a3a3fc3474f94b4be7bec051b0643 2024-11-20T16:55:24,650 INFO [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/table/891a3a3fc3474f94b4be7bec051b0643, entries=7, sequenceid=90, filesize=5.8 K 2024-11-20T16:55:24,650 INFO [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1741ms, sequenceid=90, compaction requested=false 2024-11-20T16:55:24,653 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-20T16:55:24,654 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T16:55:24,654 INFO [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T16:55:24,654 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732121722909Running coprocessor pre-close hooks at 1732121722909Disabling compacts and flushes for region at 1732121722909Disabling writes for close at 1732121722909Obtaining lock to block concurrent updates at 1732121722909Preparing flush snapshotting stores in 1588230740 at 1732121722909Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732121722910 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732121722911 (+1 ms)Flushing 1588230740/info: creating writer at 1732121722911Flushing 1588230740/info: appending metadata at 1732121722934 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732121722934Flushing 1588230740/ns: creating writer at 1732121723342 (+408 ms)Flushing 1588230740/ns: appending metadata at 1732121723359 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732121723359Flushing 1588230740/rep_barrier: creating writer at 1732121723770 (+411 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732121723787 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732121723787Flushing 1588230740/table: creating writer at 1732121724201 (+414 ms)Flushing 1588230740/table: appending metadata at 1732121724217 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732121724218 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2701167d: reopening flushed file at 1732121724631 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a4937a1: reopening flushed file at 1732121724640 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cc99d9b: reopening flushed file at 1732121724644 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a6e6467: reopening flushed file at 1732121724647 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1741ms, sequenceid=90, compaction requested=false at 1732121724650 (+3 ms)Writing region close event to WAL at 1732121724652 (+2 ms)Running coprocessor post-close hooks at 1732121724654 (+2 ms)Closed at 1732121724654 2024-11-20T16:55:24,654 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T16:55:24,712 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(976): stopping server b60eac9bda43,33929,1732121526384; all regions closed. 2024-11-20T16:55:24,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741834_1010 (size=25344) 2024-11-20T16:55:24,725 DEBUG [RS:0;b60eac9bda43:33929 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/oldWALs 2024-11-20T16:55:24,725 INFO [RS:0;b60eac9bda43:33929 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b60eac9bda43%2C33929%2C1732121526384.meta:.meta(num 1732121530456) 2024-11-20T16:55:24,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741832_1008 (size=18255296) 2024-11-20T16:55:24,729 DEBUG [RS:0;b60eac9bda43:33929 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/oldWALs 2024-11-20T16:55:24,729 INFO [RS:0;b60eac9bda43:33929 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b60eac9bda43%2C33929%2C1732121526384:(num 1732121529471) 2024-11-20T16:55:24,729 DEBUG [RS:0;b60eac9bda43:33929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T16:55:24,729 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T16:55:24,729 INFO [RS:0;b60eac9bda43:33929 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T16:55:24,729 INFO [RS:0;b60eac9bda43:33929 {}] hbase.ChoreService(370): Chore service for: regionserver/b60eac9bda43:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T16:55:24,729 INFO [RS:0;b60eac9bda43:33929 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T16:55:24,729 INFO [regionserver/b60eac9bda43:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T16:55:24,730 INFO [RS:0;b60eac9bda43:33929 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33929 2024-11-20T16:55:24,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T16:55:24,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b60eac9bda43,33929,1732121526384 2024-11-20T16:55:24,770 INFO [RS:0;b60eac9bda43:33929 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T16:55:24,771 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b60eac9bda43,33929,1732121526384] 2024-11-20T16:55:24,779 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b60eac9bda43,33929,1732121526384 already deleted, retry=false 2024-11-20T16:55:24,779 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b60eac9bda43,33929,1732121526384 expired; onlineServers=0 2024-11-20T16:55:24,779 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b60eac9bda43,33377,1732121525625' ***** 2024-11-20T16:55:24,779 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T16:55:24,779 INFO [M:0;b60eac9bda43:33377 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T16:55:24,779 INFO [M:0;b60eac9bda43:33377 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T16:55:24,780 DEBUG [M:0;b60eac9bda43:33377 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T16:55:24,780 DEBUG [M:0;b60eac9bda43:33377 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T16:55:24,780 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T16:55:24,780 DEBUG [master/b60eac9bda43:0:becomeActiveMaster-HFileCleaner.small.0-1732121529163 {}] cleaner.HFileCleaner(306): Exit Thread[master/b60eac9bda43:0:becomeActiveMaster-HFileCleaner.small.0-1732121529163,5,FailOnTimeoutGroup] 2024-11-20T16:55:24,780 DEBUG [master/b60eac9bda43:0:becomeActiveMaster-HFileCleaner.large.0-1732121529159 {}] cleaner.HFileCleaner(306): Exit Thread[master/b60eac9bda43:0:becomeActiveMaster-HFileCleaner.large.0-1732121529159,5,FailOnTimeoutGroup] 2024-11-20T16:55:24,780 INFO [M:0;b60eac9bda43:33377 {}] hbase.ChoreService(370): Chore service for: master/b60eac9bda43:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T16:55:24,780 INFO [M:0;b60eac9bda43:33377 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T16:55:24,780 DEBUG [M:0;b60eac9bda43:33377 {}] master.HMaster(1795): Stopping service threads 2024-11-20T16:55:24,780 INFO [M:0;b60eac9bda43:33377 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T16:55:24,781 INFO [M:0;b60eac9bda43:33377 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T16:55:24,781 ERROR [M:0;b60eac9bda43:33377 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:45599 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:45599,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-20T16:55:24,782 INFO [M:0;b60eac9bda43:33377 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T16:55:24,782 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T16:55:24,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T16:55:24,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T16:55:24,787 DEBUG [M:0;b60eac9bda43:33377 {}] zookeeper.ZKUtil(347): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T16:55:24,787 WARN [M:0;b60eac9bda43:33377 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T16:55:24,788 INFO [M:0;b60eac9bda43:33377 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/.lastflushedseqids 2024-11-20T16:55:24,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742495_1671 (size=72) 2024-11-20T16:55:24,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T16:55:24,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33929-0x1015a728ccd0001, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T16:55:24,881 INFO [RS:0;b60eac9bda43:33929 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T16:55:24,881 INFO [RS:0;b60eac9bda43:33929 {}] regionserver.HRegionServer(1031): Exiting; stopping=b60eac9bda43,33929,1732121526384; zookeeper connection closed. 2024-11-20T16:55:24,882 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2067e16f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2067e16f 2024-11-20T16:55:24,883 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-20T16:55:25,202 INFO [M:0;b60eac9bda43:33377 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T16:55:25,202 INFO [M:0;b60eac9bda43:33377 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T16:55:25,202 DEBUG [M:0;b60eac9bda43:33377 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T16:55:25,202 INFO [M:0;b60eac9bda43:33377 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T16:55:25,202 DEBUG [M:0;b60eac9bda43:33377 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T16:55:25,202 DEBUG [M:0;b60eac9bda43:33377 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T16:55:25,202 DEBUG [M:0;b60eac9bda43:33377 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T16:55:25,202 INFO [M:0;b60eac9bda43:33377 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=797.65 KB heapSize=978.44 KB 2024-11-20T16:55:25,222 DEBUG [M:0;b60eac9bda43:33377 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/871879525d8e47569f3a6c793ca9d948 is 82, key is hbase:meta,,1/info:regioninfo/1732121530541/Put/seqid=0 2024-11-20T16:55:25,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742496_1672 (size=5672) 2024-11-20T16:55:25,626 INFO [M:0;b60eac9bda43:33377 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2233 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/871879525d8e47569f3a6c793ca9d948 2024-11-20T16:55:25,654 DEBUG [M:0;b60eac9bda43:33377 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/255d7d625279438b84e630b49ee1fc3e is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x9B/proc:d/1732121691004/Put/seqid=0 2024-11-20T16:55:25,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742497_1673 (size=47172) 2024-11-20T16:55:25,979 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T16:55:25,980 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T16:55:25,980 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T16:55:26,058 INFO [M:0;b60eac9bda43:33377 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=797.09 KB at sequenceid=2233 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/255d7d625279438b84e630b49ee1fc3e 2024-11-20T16:55:26,062 INFO [M:0;b60eac9bda43:33377 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 255d7d625279438b84e630b49ee1fc3e 2024-11-20T16:55:26,083 DEBUG [M:0;b60eac9bda43:33377 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f6d202fd70c34b95b4ea3e0747b0d317 is 69, key is b60eac9bda43,33929,1732121526384/rs:state/1732121529204/Put/seqid=0 2024-11-20T16:55:26,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073742498_1674 (size=5156) 2024-11-20T16:55:26,487 INFO [M:0;b60eac9bda43:33377 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2233 (bloomFilter=true), to=hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f6d202fd70c34b95b4ea3e0747b0d317 2024-11-20T16:55:26,490 DEBUG [M:0;b60eac9bda43:33377 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/871879525d8e47569f3a6c793ca9d948 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/871879525d8e47569f3a6c793ca9d948 2024-11-20T16:55:26,493 INFO [M:0;b60eac9bda43:33377 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/871879525d8e47569f3a6c793ca9d948, entries=8, sequenceid=2233, filesize=5.5 K 2024-11-20T16:55:26,493 DEBUG [M:0;b60eac9bda43:33377 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/255d7d625279438b84e630b49ee1fc3e as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/255d7d625279438b84e630b49ee1fc3e 2024-11-20T16:55:26,495 INFO [M:0;b60eac9bda43:33377 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 255d7d625279438b84e630b49ee1fc3e 2024-11-20T16:55:26,495 INFO [M:0;b60eac9bda43:33377 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/255d7d625279438b84e630b49ee1fc3e, entries=180, sequenceid=2233, filesize=46.1 K 2024-11-20T16:55:26,496 DEBUG [M:0;b60eac9bda43:33377 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f6d202fd70c34b95b4ea3e0747b0d317 as hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f6d202fd70c34b95b4ea3e0747b0d317 2024-11-20T16:55:26,498 INFO [M:0;b60eac9bda43:33377 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45599/user/jenkins/test-data/e1a008e9-e878-6b57-311c-f0b40e657625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f6d202fd70c34b95b4ea3e0747b0d317, entries=1, sequenceid=2233, filesize=5.0 K 2024-11-20T16:55:26,499 INFO [M:0;b60eac9bda43:33377 {}] regionserver.HRegion(3140): Finished flush of dataSize ~797.65 KB/816790, heapSize ~978.14 KB/1001616, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1296ms, sequenceid=2233, compaction requested=false 2024-11-20T16:55:26,500 INFO [M:0;b60eac9bda43:33377 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T16:55:26,500 DEBUG [M:0;b60eac9bda43:33377 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732121725202Disabling compacts and flushes for region at 1732121725202Disabling writes for close at 1732121725202Obtaining lock to block concurrent updates at 1732121725202Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732121725202Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=816790, getHeapSize=1001856, getOffHeapSize=0, getCellsCount=2438 at 1732121725203 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732121725203Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732121725204 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732121725221 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732121725221Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732121725631 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732121725654 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732121725654Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732121726063 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732121726083 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732121726083Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c769c11: reopening flushed file at 1732121726489 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f9b3927: reopening flushed file at 1732121726493 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45759038: reopening flushed file at 1732121726496 (+3 ms)Finished flush of dataSize ~797.65 KB/816790, heapSize ~978.14 KB/1001616, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1296ms, sequenceid=2233, compaction requested=false at 1732121726499 (+3 ms)Writing region close event to WAL at 1732121726500 (+1 ms)Closed at 1732121726500 2024-11-20T16:55:26,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35935 is added to blk_1073741830_1006 (size=963174) 2024-11-20T16:55:26,503 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T16:55:26,503 INFO [M:0;b60eac9bda43:33377 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T16:55:26,503 INFO [M:0;b60eac9bda43:33377 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33377 2024-11-20T16:55:26,503 INFO [M:0;b60eac9bda43:33377 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T16:55:26,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T16:55:26,654 INFO [M:0;b60eac9bda43:33377 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T16:55:26,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1015a728ccd0000, quorum=127.0.0.1:58094, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T16:55:26,662 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T16:55:26,665 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T16:55:26,665 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T16:55:26,665 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T16:55:26,666 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/hadoop.log.dir/,STOPPED} 2024-11-20T16:55:26,670 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T16:55:26,670 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T16:55:26,670 WARN [BP-2108073788-172.17.0.2-1732121522165 heartbeating to localhost/127.0.0.1:45599 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T16:55:26,670 WARN [BP-2108073788-172.17.0.2-1732121522165 heartbeating to localhost/127.0.0.1:45599 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2108073788-172.17.0.2-1732121522165 (Datanode Uuid d89f575f-a757-488e-8350-da815eb7d10e) service to localhost/127.0.0.1:45599 2024-11-20T16:55:26,674 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/cluster_e1fea354-106b-cccb-72d5-7b2ff6b355d1/data/data1/current/BP-2108073788-172.17.0.2-1732121522165 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T16:55:26,675 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/cluster_e1fea354-106b-cccb-72d5-7b2ff6b355d1/data/data2/current/BP-2108073788-172.17.0.2-1732121522165 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T16:55:26,675 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T16:55:26,687 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T16:55:26,688 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T16:55:26,688 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T16:55:26,688 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T16:55:26,688 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3e23fff1-1350-0844-f509-98a49314fe74/hadoop.log.dir/,STOPPED} 2024-11-20T16:55:26,703 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T16:55:26,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down